From b8fb56a75c4c15b9ae98f3dace3b1ec563d1c1bc Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Thu, 13 Jun 2024 11:07:06 +0200 Subject: [PATCH 01/21] wip hooks --- .gitignore | 1 + Dockerfile | 1 + config/settings-hooks.yaml.template | 10 +++++ docker-compose.yaml | 30 ++++++++++++++ fedn/cli/__init__.py | 1 + fedn/cli/hooks_cmd.py | 63 +++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+) create mode 100644 config/settings-hooks.yaml.template create mode 100644 fedn/cli/hooks_cmd.py diff --git a/.gitignore b/.gitignore index b595e49c9..e75a5b9e9 100644 --- a/.gitignore +++ b/.gitignore @@ -177,6 +177,7 @@ config/extra-hosts-reducer.yaml config/settings-client.yaml config/settings-reducer.yaml config/settings-combiner.yaml +config/settings-hooks.yaml ./tmp/* diff --git a/Dockerfile b/Dockerfile index 49c91a3be..866e8d812 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,7 @@ ARG REQUIREMENTS="" COPY . /app COPY config/settings-client.yaml.template /app/config/settings-client.yaml COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml +COPY config/settings-hooks.yaml.template /app/config/settings-hooks.yaml COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml COPY $REQUIREMENTS /app/config/requirements.txt diff --git a/config/settings-hooks.yaml.template b/config/settings-hooks.yaml.template new file mode 100644 index 000000000..33a50d604 --- /dev/null +++ b/config/settings-hooks.yaml.template @@ -0,0 +1,10 @@ +network_id: fedn-network +discover_host: api-server +discover_port: 8092 + +name: hooks +host: hooks +port: 12081 +max_clients: 30 + + diff --git a/docker-compose.yaml b/docker-compose.yaml index 85386c6da..63d7e69dd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -113,6 +113,36 @@ services: depends_on: - api-server + hooks: + environment: + - PYTHONUNBUFFERED=0 + - GET_HOSTS_FROM=dns + build: + context: . + args: + BASE_IMG: ${BASE_IMG:-python:3.10-slim} + GRPC_HEALTH_PROBE_VERSION: v0.4.24 + working_dir: /app + volumes: + - ${HOST_REPO_DIR:-.}/user-hooks:/app/user-hooks + entrypoint: [ "sh", "-c" ] + command: + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn hooks start --init config/settings-user-hooks.yaml" + ports: + - 12081:12081 + healthcheck: + test: + [ + "CMD", + "/bin/grpc_health_probe", + "-addr=localhost:12081" + ] + interval: 20s + timeout: 10s + retries: 5 + depends_on: + - combiner + # Client client: environment: diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 137fc9b9c..f4be1a9ae 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -1,6 +1,7 @@ from .client_cmd import client_cmd # noqa: F401 from .combiner_cmd import combiner_cmd # noqa: F401 from .config_cmd import config_cmd # noqa: F401 +from .hooks_cmd import hooks_cmd # noqa: F401 from .main import main # noqa: F401 from .model_cmd import model_cmd # noqa: F401 from .package_cmd import package_cmd # noqa: F401 diff --git a/fedn/cli/hooks_cmd.py b/fedn/cli/hooks_cmd.py new file mode 100644 index 000000000..8d584f52c --- /dev/null +++ b/fedn/cli/hooks_cmd.py @@ -0,0 +1,63 @@ +import uuid + +import click +import requests + +from fedn.network.combiner.combiner import Combiner + +from .main import main +from .shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response + + +@main.group("hooks") +@click.pass_context +def hooks_cmd(ctx): + """:param ctx:""" + pass + + +@hooks_cmd.command("start") +@click.option("-d", "--discoverhost", required=False, help="Hostname for discovery services (reducer).") +@click.option("-p", "--discoverport", required=False, help="Port for discovery services (reducer).") +@click.option("-t", "--token", required=False, help="Set token provided by reducer if enabled") +@click.option("-n", "--name", required=False, default="combiner" + str(uuid.uuid4())[:8], help="Set name for combiner.") +@click.option("-h", "--host", required=False, default="combiner", help="Set hostname.") +@click.option("-i", "--port", required=False, default=12080, help="Set port.") +@click.option("-f", "--fqdn", required=False, default=None, help="Set fully qualified domain name") +@click.option("-s", "--secure", is_flag=True, help="Enable SSL/TLS encrypted gRPC channels.") +@click.option("-v", "--verify", is_flag=True, help="Verify SSL/TLS for REST discovery service (reducer)") +@click.option("-c", "--max_clients", required=False, default=30, help="The maximal number of client connections allowed.") +@click.option("-in", "--init", required=False, default=None, help="Path to configuration file to (re)init combiner.") +@click.pass_context +def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, secure, verify, max_clients, init): + """:param ctx: + :param discoverhost: + :param discoverport: + :param token: + :param name: + :param hostname: + :param port: + :param secure: + :param max_clients: + :param init: + """ + config = { + "discover_host": discoverhost, + "discover_port": discoverport, + "token": token, + "host": host, + "port": port, + "fqdn": fqdn, + "name": name, + "secure": secure, + "verify": verify, + "max_clients": max_clients, + } + click.echo("started hooks container") + if init: + apply_config(init, config) + click.echo(f"\nCombiner configuration loaded from file: {init}") + click.echo("Values set in file override defaults and command line arguments...\n") + + # combiner = Combiner(config) + # combiner.run() From 7449c66b2a7bc4ed7c1bebfe176475ce98f0ca2f Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Fri, 26 Jul 2024 09:19:18 +0200 Subject: [PATCH 02/21] working --- config/settings-hooks.yaml.template | 4 +- docker-compose.yaml | 12 +-- examples/mnist-pytorch/client/aggregator.py | 21 +++++ fedn/cli/hooks_cmd.py | 53 ++---------- fedn/network/api/client.py | 10 +++ fedn/network/api/interface.py | 19 +++-- fedn/network/api/server.py | 3 +- fedn/network/combiner/aggregators/custom.py | 93 ++++++++++++++++++++ fedn/network/combiner/combiner.py | 21 ++++- fedn/network/combiner/hook_client.py | 40 +++++++++ fedn/network/combiner/interfaces.py | 22 +++++ fedn/network/combiner/roundhandler.py | 7 ++ fedn/network/controller/control.py | 2 + fedn/network/grpc/fedn.proto | 19 +++++ fedn/network/grpc/fedn_pb2.py | 54 ++++++------ fedn/network/grpc/fedn_pb2_grpc.py | 94 +++++++++++++++++++++ fedn/network/hooks/__init__.py | 1 + fedn/network/hooks/hooks.py | 76 +++++++++++++++++ fedn/network/hooks/safe_builtins.py | 74 ++++++++++++++++ pyproject.toml | 3 +- 20 files changed, 537 insertions(+), 91 deletions(-) create mode 100644 examples/mnist-pytorch/client/aggregator.py create mode 100644 fedn/network/combiner/aggregators/custom.py create mode 100644 fedn/network/combiner/hook_client.py create mode 100644 fedn/network/hooks/__init__.py create mode 100644 fedn/network/hooks/hooks.py create mode 100644 fedn/network/hooks/safe_builtins.py diff --git a/config/settings-hooks.yaml.template b/config/settings-hooks.yaml.template index 33a50d604..e395b20ce 100644 --- a/config/settings-hooks.yaml.template +++ b/config/settings-hooks.yaml.template @@ -5,6 +5,4 @@ discover_port: 8092 name: hooks host: hooks port: 12081 -max_clients: 30 - - +max_clients: 30 \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 63d7e69dd..c22f60283 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -87,6 +87,7 @@ services: environment: - PYTHONUNBUFFERED=0 - GET_HOSTS_FROM=dns + - HOOK_SERVICE_HOST=hook:12081 build: context: . args: @@ -112,10 +113,11 @@ services: retries: 5 depends_on: - api-server - + - hooks + # Hooks hooks: + container_name: hook environment: - - PYTHONUNBUFFERED=0 - GET_HOSTS_FROM=dns build: context: . @@ -124,10 +126,10 @@ services: GRPC_HEALTH_PROBE_VERSION: v0.4.24 working_dir: /app volumes: - - ${HOST_REPO_DIR:-.}/user-hooks:/app/user-hooks + - ${HOST_REPO_DIR:-.}/fedn:/app/fedn entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn hooks start --init config/settings-user-hooks.yaml" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn hooks start" ports: - 12081:12081 healthcheck: @@ -140,8 +142,6 @@ services: interval: 20s timeout: 10s retries: 5 - depends_on: - - combiner # Client client: diff --git a/examples/mnist-pytorch/client/aggregator.py b/examples/mnist-pytorch/client/aggregator.py new file mode 100644 index 000000000..65d829429 --- /dev/null +++ b/examples/mnist-pytorch/client/aggregator.py @@ -0,0 +1,21 @@ +import numpy as np + + +class FunctionProvider: + def __init__(self) -> None: + pass + + def aggregate(self, parameters): + if len(parameters) == 0: + return [] + num_clients = len(parameters) + + summed_parameters = [np.zeros_like(param) for param in parameters[0]] + + for client_params in parameters: + for i, param in enumerate(client_params): + summed_parameters[i] += param + + averaged_parameters = [param / num_clients for param in summed_parameters] + + return averaged_parameters diff --git a/fedn/cli/hooks_cmd.py b/fedn/cli/hooks_cmd.py index 8d584f52c..e3515bc5a 100644 --- a/fedn/cli/hooks_cmd.py +++ b/fedn/cli/hooks_cmd.py @@ -1,12 +1,8 @@ -import uuid - import click -import requests -from fedn.network.combiner.combiner import Combiner +from fedn.network.hooks.hooks import serve from .main import main -from .shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response @main.group("hooks") @@ -17,47 +13,8 @@ def hooks_cmd(ctx): @hooks_cmd.command("start") -@click.option("-d", "--discoverhost", required=False, help="Hostname for discovery services (reducer).") -@click.option("-p", "--discoverport", required=False, help="Port for discovery services (reducer).") -@click.option("-t", "--token", required=False, help="Set token provided by reducer if enabled") -@click.option("-n", "--name", required=False, default="combiner" + str(uuid.uuid4())[:8], help="Set name for combiner.") -@click.option("-h", "--host", required=False, default="combiner", help="Set hostname.") -@click.option("-i", "--port", required=False, default=12080, help="Set port.") -@click.option("-f", "--fqdn", required=False, default=None, help="Set fully qualified domain name") -@click.option("-s", "--secure", is_flag=True, help="Enable SSL/TLS encrypted gRPC channels.") -@click.option("-v", "--verify", is_flag=True, help="Verify SSL/TLS for REST discovery service (reducer)") -@click.option("-c", "--max_clients", required=False, default=30, help="The maximal number of client connections allowed.") -@click.option("-in", "--init", required=False, default=None, help="Path to configuration file to (re)init combiner.") @click.pass_context -def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, secure, verify, max_clients, init): - """:param ctx: - :param discoverhost: - :param discoverport: - :param token: - :param name: - :param hostname: - :param port: - :param secure: - :param max_clients: - :param init: - """ - config = { - "discover_host": discoverhost, - "discover_port": discoverport, - "token": token, - "host": host, - "port": port, - "fqdn": fqdn, - "name": name, - "secure": secure, - "verify": verify, - "max_clients": max_clients, - } - click.echo("started hooks container") - if init: - apply_config(init, config) - click.echo(f"\nCombiner configuration loaded from file: {init}") - click.echo("Values set in file override defaults and command line arguments...\n") - - # combiner = Combiner(config) - # combiner.run() +def start_cmd(ctx): + """:param ctx:""" + click.echo("Started hooks container") + serve() diff --git a/fedn/network/api/client.py b/fedn/network/api/client.py index d26be03ff..e63297d19 100644 --- a/fedn/network/api/client.py +++ b/fedn/network/api/client.py @@ -574,6 +574,7 @@ def start_session( helper: str = "", min_clients: int = 1, requested_clients: int = 8, + function_provider_path: str = None, ): """Start a new session. @@ -617,6 +618,7 @@ def start_session( "helper": helper, "min_clients": min_clients, "requested_clients": requested_clients, + "function_provider": function_provider_path if function_provider_path is None else self._read_function_provider(function_provider_path), }, verify=self.verify, headers=self.headers, @@ -787,3 +789,11 @@ def get_validations_count(self): _json = response.json() return _json + + def _read_function_provider(self, path): + # Open the file in read mode + with open(path, "r") as file: + file_contents = file.read() + + # Print the file contents + return file_contents diff --git a/fedn/network/api/interface.py b/fedn/network/api/interface.py index 5cd465085..655e4bef8 100644 --- a/fedn/network/api/interface.py +++ b/fedn/network/api/interface.py @@ -11,6 +11,7 @@ from fedn.common.config import get_controller_config, get_network_config from fedn.common.log_config import logger from fedn.network.combiner.interfaces import CombinerInterface, CombinerUnavailableError +from fedn.network.combiner.modelservice import load_model_from_BytesIO from fedn.network.state import ReducerState, ReducerStateToString from fedn.utils.checksum import sha from fedn.utils.plots import Plot @@ -649,13 +650,17 @@ def set_initial_model(self, file): :rtype: :class:`flask.Response` """ try: - object = BytesIO() - object.seek(0, 0) - file.seek(0) - object.write(file.read()) helper = self.control.get_helper() - object.seek(0) - model = helper.load(object) + + # Read file data into a BytesIO object + file_bytes = BytesIO() + file.seek(0) + file_bytes.write(file.read()) + file_bytes.seek(0) + + # Load the model using the load_model_from_BytesIO function + model_bytes = file_bytes.read() + model = load_model_from_BytesIO(model_bytes, helper) self.control.commit(file.filename, model) except Exception as e: logger.debug(e) @@ -966,6 +971,7 @@ def start_session( helper="", min_clients=1, requested_clients=8, + function_provider=None, ): """Start a session. @@ -1068,6 +1074,7 @@ def start_session( "task": (""), "validate": validate, "helper_type": helper, + "function_provider": function_provider, } # Start session diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 8f046ee80..955a43c1d 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -5,9 +5,8 @@ from fedn.common.config import get_controller_config from fedn.network.api.auth import jwt_auth_required from fedn.network.api.interface import API +from fedn.network.api.shared import control, statestore from fedn.network.api.v1 import _routes -from fedn.network.api.shared import statestore, control - custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) api = API(statestore, control) diff --git a/fedn/network/combiner/aggregators/custom.py b/fedn/network/combiner/aggregators/custom.py new file mode 100644 index 000000000..ab0270c67 --- /dev/null +++ b/fedn/network/combiner/aggregators/custom.py @@ -0,0 +1,93 @@ +import traceback + +import numpy as np + +from fedn.common.log_config import logger +from fedn.network.combiner.aggregators.aggregatorbase import AggregatorBase + + +class Aggregator(AggregatorBase): + """Custom aggregator provided from user defined code. + + :param id: A reference to id of :class: `fedn.network.combiner.Combiner` + :type id: str + :param storage: Model repository for :class: `fedn.network.combiner.Combiner` + :type storage: class: `fedn.common.storage.s3.s3repo.S3ModelRepository` + :param server: A handle to the Combiner class :class: `fedn.network.combiner.Combiner` + :type server: class: `fedn.network.combiner.Combiner` + :param modelservice: A handle to the model service :class: `fedn.network.combiner.modelservice.ModelService` + :type modelservice: class: `fedn.network.combiner.modelservice.ModelService` + :param control: A handle to the :class: `fedn.network.combiner.roundhandler.RoundHandler` + :type control: class: `fedn.network.combiner.roundhandler.RoundHandler` + + """ + + def __init__(self, storage, server, modelservice, round_handler): + """Constructor method""" + super().__init__(storage, server, modelservice, round_handler) + + self.name = "custom" + self.code_set = False + + def combine_models(self, helper=None, delete_models=True, parameters=None): + """Aggregate all model updates with custom aggregator. + + :param helper: An instance of :class: `fedn.utils.helpers.helpers.HelperBase`, ML framework specific helper, defaults to None + :type helper: class: `fedn.utils.helpers.helpers.HelperBase`, optional + :param time_window: The time window for model aggregation, defaults to 180 + :type time_window: int, optional + :param max_nr_models: The maximum number of updates aggregated, defaults to 100 + :type max_nr_models: int, optional + :param delete_models: Delete models from storage after aggregation, defaults to True + :type delete_models: bool, optional + :return: The global model and metadata + :rtype: tuple + """ + data = {} + data["time_model_load"] = 0.0 + data["time_model_aggregation"] = 0.0 + + model = None + nr_aggregated_models = 0 + total_examples = 0 + + logger.info("AGGREGATOR({}): Aggregating model updates... ".format(self.name)) + if not self.code_set: + self.round_handler.set_function_provider() + self.code_set = True + while not self.model_updates.empty(): + try: + # Get next model from queue + logger.info("AGGREGATOR({}): Getting next model update from queue.".format(self.name)) + model_update = self.next_model_update() + + # Load model parameters and metadata + logger.info("AGGREGATOR({}): Loading model metadata {}.".format(self.name, model_update.model_update_id)) + model_next, metadata = self.load_model_update(model_update, helper) + + logger.info("AGGREGATOR({}): Processing model update {}, metadata: {} ".format(self.name, model_update.model_update_id, metadata)) + + # Increment total number of examples + total_examples += metadata["num_examples"] + + nr_aggregated_models += 1 + # Delete model from storage + if delete_models: + self.modelservice.temp_model_storage.delete(model_update.model_update_id) + logger.info("AGGREGATOR({}): Deleted model update {} from storage.".format(self.name, model_update.model_update_id)) + + self.model_updates.task_done() + if not self.model_updates.empty(): + self.round_handler.combiner_hook_client.call_function("store_parameters", model_next, helper) + else: + model = self.round_handler.combiner_hook_client.call_function("aggregate", model_next, helper) + except Exception as e: + tb = traceback.format_exc() + logger.error(f"AGGREGATOR({self.name}): Error encoutered while processing model update: {e}") + logger.error(tb) + self.model_updates.task_done() + + data["nr_aggregated_models"] = nr_aggregated_models + + logger.info("AGGREGATOR({}): Aggregation completed, aggregated {} models.".format(self.name, nr_aggregated_models)) + return model, data diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index 70755ac6b..22f4bc807 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -466,7 +466,6 @@ def SetAggregator(self, control: fedn.ControlRequest, context): logger.debug("grpc.Combiner.SetAggregator: Called") for parameter in control.parameter: aggregator = parameter.value - status = self.round_handler.set_aggregator(aggregator) response = fedn.ControlResponse() @@ -474,7 +473,27 @@ def SetAggregator(self, control: fedn.ControlRequest, context): response.message = "Success" else: response.message = "Failed" + return response + + def SetFunctionProvider(self, control: fedn.ControlRequest, context): + """Set a function provider. + + :param control: the control request + :type control: :class:`fedn.network.grpc.fedn_pb2.ControlRequest` + :param context: the context (unused) + :type context: :class:`grpc._server._Context` + :return: the control response + :rtype: :class:`fedn.network.grpc.fedn_pb2.ControlResponse` + """ + logger.debug("grpc.Combiner.SetFunctionProvider: Called") + for parameter in control.parameter: + function_provider = parameter.value + + self.round_handler.function_provider_code = function_provider + response = fedn.ControlResponse() + response.message = "Success" + logger.info(f"set function provider response {response}") return response def FlushAggregationQueue(self, control: fedn.ControlRequest, context): diff --git a/fedn/network/combiner/hook_client.py b/fedn/network/combiner/hook_client.py new file mode 100644 index 000000000..422c5cb94 --- /dev/null +++ b/fedn/network/combiner/hook_client.py @@ -0,0 +1,40 @@ +import os + +import grpc + +import fedn.network.grpc.fedn_pb2 as fedn +import fedn.network.grpc.fedn_pb2_grpc as rpc +from fedn.common.log_config import logger +from fedn.network.combiner.modelservice import load_model_from_BytesIO, serialize_model_to_BytesIO + + +class CombinerHookClient: + def __init__(self): + logger.info("Starting hook client") + self.hook_service_host = os.getenv("HOOK_SERVICE_HOST", "hook:12081") + self.channel = grpc.insecure_channel(self.hook_service_host) + self.stub = rpc.FunctionServiceStub(self.channel) + + def call_function_service(self, task, payload): + request = fedn.FunctionRequest(task=task, payload_string=payload) if task == "setup" else fedn.FunctionRequest(task=task, payload_bytes=payload) + try: + response = self.stub.ExecuteFunction(request) + return response + except grpc.RpcError as e: + logger.info(f"RPC failed: {e}") + return None + + # Example method to trigger function execution + def set_function_provider(self, class_code): + if not isinstance(class_code, str): + raise TypeError("class_code must be of type string") + self.call_function_service("setup", class_code) + + def call_function(self, task, payload, helper): + if task == "aggregate": + payload = serialize_model_to_BytesIO(payload, helper).getvalue() + response = self.call_function_service(task, payload) + return load_model_from_BytesIO(response.result_bytes, helper) + if task == "store_parameters": + payload = serialize_model_to_BytesIO(payload, helper).getvalue() + response = self.call_function_service(task, payload) diff --git a/fedn/network/combiner/interfaces.py b/fedn/network/combiner/interfaces.py index 935b75442..32c8dbf41 100644 --- a/fedn/network/combiner/interfaces.py +++ b/fedn/network/combiner/interfaces.py @@ -203,6 +203,28 @@ def set_aggregator(self, aggregator): else: raise + def set_function_provider(self, function_provider): + """Set the function provider module. + + :param function provider: Stringified function provider code. + :type config: str + """ + channel = Channel(self.address, self.port, self.certificate).get_channel() + control = rpc.ControlStub(channel) + + request = fedn.ControlRequest() + p = request.parameter.add() + p.key = "function_provider" + p.value = function_provider + + try: + control.SetFunctionProvider(request) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.UNAVAILABLE: + raise CombinerUnavailableError + else: + raise + def submit(self, config: RoundConfig): """Submit a compute plan to the combiner. diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index ef9029de9..b99a7c524 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -8,6 +8,7 @@ from fedn.common.log_config import logger from fedn.network.combiner.aggregators.aggregatorbase import get_aggregator +from fedn.network.combiner.hook_client import CombinerHookClient from fedn.network.combiner.modelservice import load_model_from_BytesIO, serialize_model_to_BytesIO from fedn.utils.helpers.helpers import get_helper from fedn.utils.parameters import Parameters @@ -94,6 +95,12 @@ def __init__(self, storage, server, modelservice): def set_aggregator(self, aggregator): self.aggregator = get_aggregator(aggregator, self.storage, self.server, self.modelservice, self) + def set_function_provider(self): + if not hasattr(self, "function_provider_code") or self.function_provider_code is None: + raise Exception("Custom function provider code need to be set.") + self.combiner_hook_client = CombinerHookClient() + self.combiner_hook_client.set_function_provider(self.function_provider_code) + def push_round_config(self, round_config: RoundConfig) -> str: """Add a round_config (job description) to the inbox. diff --git a/fedn/network/controller/control.py b/fedn/network/controller/control.py index c7a6d1c26..4fe331007 100644 --- a/fedn/network/controller/control.py +++ b/fedn/network/controller/control.py @@ -161,6 +161,8 @@ def session(self, config: RoundConfig) -> None: for combiner in self.network.get_combiners(): combiner.set_aggregator(config["aggregator"]) + if config["function_provider"] is not None: + combiner.set_function_provider(config["function_provider"]) self.set_session_status(config["session_id"], "Started") # Execute the rounds in this session diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index fd2f1d5c5..cefa1728b 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -194,6 +194,7 @@ service Control { rpc Stop(ControlRequest) returns (ControlResponse); rpc FlushAggregationQueue(ControlRequest) returns (ControlResponse); rpc SetAggregator(ControlRequest) returns (ControlResponse); + rpc SetFunctionProvider(ControlRequest) returns (ControlResponse); } service Reducer { @@ -243,3 +244,21 @@ service Combiner { } +message FunctionRequest { + string task = 1; + oneof payload { + string payload_string = 2; + bytes payload_bytes = 3; + } +} + +message FunctionResponse { + oneof result { + string result_string = 2; + bytes result_bytes = 3; + } +} + +service FunctionService { + rpc ExecuteFunction(FunctionRequest) returns (FunctionResponse); +} diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index 714bef0e5..b763a7c30 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: fedn/network/grpc/fedn.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -14,26 +15,25 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"0\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"0\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"]\n\x0f\x46unctionRequest\x12\x0c\n\x04task\x18\x01 \x01(\t\x12\x18\n\x0epayload_string\x18\x02 \x01(\tH\x00\x12\x17\n\rpayload_bytes\x18\x03 \x01(\x0cH\x00\x42\t\n\x07payload\"M\n\x10\x46unctionResponse\x12\x17\n\rresult_string\x18\x02 \x01(\tH\x00\x12\x16\n\x0cresult_bytes\x18\x03 \x01(\x0cH\x00\x42\x08\n\x06result*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbc\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x42\n\x13SetFunctionProvider\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2S\n\x0f\x46unctionService\x12@\n\x0f\x45xecuteFunction\x12\x15.fedn.FunctionRequest\x1a\x16.fedn.FunctionResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'fedn.network.grpc.fedn_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _globals['_STATUSTYPE']._serialized_start=2313 - _globals['_STATUSTYPE']._serialized_end=2445 - _globals['_QUEUE']._serialized_start=2447 - _globals['_QUEUE']._serialized_end=2483 - _globals['_MODELSTATUS']._serialized_start=2485 - _globals['_MODELSTATUS']._serialized_end=2568 - _globals['_ROLE']._serialized_start=2570 - _globals['_ROLE']._serialized_end=2626 - _globals['_COMMAND']._serialized_start=2628 - _globals['_COMMAND']._serialized_end=2702 - _globals['_CONNECTIONSTATUS']._serialized_start=2704 - _globals['_CONNECTIONSTATUS']._serialized_end=2777 + _globals['_STATUSTYPE']._serialized_start=2487 + _globals['_STATUSTYPE']._serialized_end=2619 + _globals['_QUEUE']._serialized_start=2621 + _globals['_QUEUE']._serialized_end=2657 + _globals['_MODELSTATUS']._serialized_start=2659 + _globals['_MODELSTATUS']._serialized_end=2742 + _globals['_ROLE']._serialized_start=2744 + _globals['_ROLE']._serialized_end=2800 + _globals['_COMMAND']._serialized_start=2802 + _globals['_COMMAND']._serialized_end=2876 + _globals['_CONNECTIONSTATUS']._serialized_start=2878 + _globals['_CONNECTIONSTATUS']._serialized_end=2951 _globals['_RESPONSE']._serialized_start=71 _globals['_RESPONSE']._serialized_end=129 _globals['_STATUS']._serialized_start=132 @@ -78,14 +78,20 @@ _globals['_CONNECTIONREQUEST']._serialized_end=2248 _globals['_CONNECTIONRESPONSE']._serialized_start=2250 _globals['_CONNECTIONRESPONSE']._serialized_end=2310 - _globals['_MODELSERVICE']._serialized_start=2779 - _globals['_MODELSERVICE']._serialized_end=2901 - _globals['_CONTROL']._serialized_start=2904 - _globals['_CONTROL']._serialized_end=3152 - _globals['_REDUCER']._serialized_start=3154 - _globals['_REDUCER']._serialized_end=3240 - _globals['_CONNECTOR']._serialized_start=3243 - _globals['_CONNECTOR']._serialized_end=3670 - _globals['_COMBINER']._serialized_start=3673 - _globals['_COMBINER']._serialized_end=3864 + _globals['_FUNCTIONREQUEST']._serialized_start=2312 + _globals['_FUNCTIONREQUEST']._serialized_end=2405 + _globals['_FUNCTIONRESPONSE']._serialized_start=2407 + _globals['_FUNCTIONRESPONSE']._serialized_end=2484 + _globals['_MODELSERVICE']._serialized_start=2953 + _globals['_MODELSERVICE']._serialized_end=3075 + _globals['_CONTROL']._serialized_start=3078 + _globals['_CONTROL']._serialized_end=3394 + _globals['_REDUCER']._serialized_start=3396 + _globals['_REDUCER']._serialized_end=3482 + _globals['_CONNECTOR']._serialized_start=3485 + _globals['_CONNECTOR']._serialized_end=3912 + _globals['_COMBINER']._serialized_start=3915 + _globals['_COMBINER']._serialized_end=4106 + _globals['_FUNCTIONSERVICE']._serialized_start=4108 + _globals['_FUNCTIONSERVICE']._serialized_end=4191 # @@protoc_insertion_point(module_scope) diff --git a/fedn/network/grpc/fedn_pb2_grpc.py b/fedn/network/grpc/fedn_pb2_grpc.py index 3c2db9a2c..63bf1f625 100644 --- a/fedn/network/grpc/fedn_pb2_grpc.py +++ b/fedn/network/grpc/fedn_pb2_grpc.py @@ -128,6 +128,11 @@ def __init__(self, channel): request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, ) + self.SetFunctionProvider = channel.unary_unary( + '/fedn.Control/SetFunctionProvider', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + ) class ControlServicer(object): @@ -157,6 +162,12 @@ def SetAggregator(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def SetFunctionProvider(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_ControlServicer_to_server(servicer, server): rpc_method_handlers = { @@ -180,6 +191,11 @@ def add_ControlServicer_to_server(servicer, server): request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, ), + 'SetFunctionProvider': grpc.unary_unary_rpc_method_handler( + servicer.SetFunctionProvider, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'fedn.Control', rpc_method_handlers) @@ -258,6 +274,23 @@ def SetAggregator(request, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod + def SetFunctionProvider(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/fedn.Control/SetFunctionProvider', + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + class ReducerStub(object): """Missing associated documentation comment in .proto file.""" @@ -710,3 +743,64 @@ def SendModelValidation(request, fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + +class FunctionServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ExecuteFunction = channel.unary_unary( + '/fedn.FunctionService/ExecuteFunction', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionResponse.FromString, + ) + + +class FunctionServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def ExecuteFunction(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_FunctionServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ExecuteFunction': grpc.unary_unary_rpc_method_handler( + servicer.ExecuteFunction, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'fedn.FunctionService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class FunctionService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def ExecuteFunction(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/fedn.FunctionService/ExecuteFunction', + fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/fedn/network/hooks/__init__.py b/fedn/network/hooks/__init__.py new file mode 100644 index 000000000..e2f53ea23 --- /dev/null +++ b/fedn/network/hooks/__init__.py @@ -0,0 +1 @@ +"""The FEDn Hooks package responsible for executing user defined code on the server.""" diff --git a/fedn/network/hooks/hooks.py b/fedn/network/hooks/hooks.py new file mode 100644 index 000000000..290720d96 --- /dev/null +++ b/fedn/network/hooks/hooks.py @@ -0,0 +1,76 @@ +from concurrent import futures +from io import BytesIO + +import grpc + +import fedn.network.grpc.fedn_pb2 as fedn +import fedn.network.grpc.fedn_pb2_grpc as rpc +from fedn.common.log_config import logger +from fedn.network.combiner.modelservice import load_model_from_BytesIO, serialize_model_to_BytesIO +from fedn.network.hooks.safe_builtins import safe_builtins +from fedn.utils.helpers.plugins.numpyhelper import Helper + +CHUNK_SIZE = 1024 * 1024 +VALID_NAME_REGEX = "^[a-zA-Z0-9_-]*$" + + +class FunctionServiceServicer(rpc.FunctionServiceServicer): + def __init__(self) -> None: + super().__init__() + self.safe_builtins = safe_builtins + + self.globals_dict = { + "__builtins__": self.safe_builtins, + } + self.helper = Helper() + self.parameters = [] + + def ExecuteFunction(self, request, context): + # Compile the function code + if request.task == "setup": + logger.info("Adding function provider.") + self.init_hook_object(request.payload_string) + return fedn.FunctionResponse(result_string="Instansiated hook functions.") + if request.task == "store_parameters": + logger.info("Executing aggregate function.") + payload = load_model_from_BytesIO(request.payload_bytes, self.helper) + self.parameters.append(payload) + return fedn.FunctionResponse(result_string="Stored parameters") + if request.task == "aggregate": + logger.info("Executing aggregate function.") + payload = load_model_from_BytesIO(request.payload_bytes, self.helper) + self.parameters.append(payload) + result = self.execute_function_code() + result_bytes = serialize_model_to_BytesIO(result, self.helper).getvalue() + return fedn.FunctionResponse(result_bytes=result_bytes) + + def init_hook_object(self, class_code): + # Prepare the globals dictionary with restricted builtins + + # Compile and execute the class code + exec(class_code, self.globals_dict) # noqa: S102 + # Instantiate the object within restricted scope + instance_code = """ +function_provider = FunctionProvider() +""" + # Compile and execute the instance code + exec(instance_code, self.globals_dict) # noqa: S102 + + def execute_function_code(self): + if not hasattr(self, "globals_dict"): + raise AttributeError("Function provider code has not been provided.") + self.globals_dict["parameters"] = self.parameters + instance_code = """ +res = function_provider.aggregate(parameters) +""" + exec(instance_code, self.globals_dict) # noqa: S102 + self.parameters = [] + return self.globals_dict["res"] + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + rpc.add_FunctionServiceServicer_to_server(FunctionServiceServicer(), server) + server.add_insecure_port("[::]:12081") + server.start() + server.wait_for_termination() diff --git a/fedn/network/hooks/safe_builtins.py b/fedn/network/hooks/safe_builtins.py new file mode 100644 index 000000000..bc8bd0aa9 --- /dev/null +++ b/fedn/network/hooks/safe_builtins.py @@ -0,0 +1,74 @@ +import builtins + +safe_builtins = { + "abs": abs, + "all": all, + "any": any, + "ascii": ascii, + "bin": bin, + "bool": bool, + "bytearray": bytearray, + "bytes": bytes, + "callable": callable, + "chr": chr, + "complex": complex, + "dict": dict, + "divmod": divmod, + "enumerate": enumerate, + "filter": filter, + "float": float, + "format": format, + "frozenset": frozenset, + "getattr": getattr, + "hasattr": hasattr, + "hash": hash, + "hex": hex, + "id": id, + "int": int, + "isinstance": isinstance, + "issubclass": issubclass, + "iter": iter, + "len": len, + "list": list, + "map": map, + "max": max, + "min": min, + "next": next, + "object": object, + "oct": oct, + "ord": ord, + "pow": pow, + "print": print, + "property": property, + "range": range, + "repr": repr, + "reversed": reversed, + "round": round, + "set": set, + "slice": slice, + "sorted": sorted, + "staticmethod": staticmethod, + "str": str, + "sum": sum, + "super": super, + "tuple": tuple, + "type": type, + "zip": zip, + "__build_class__": builtins.__build_class__, + "__name__": builtins.__name__, + "__import__": builtins.__import__, + # Adding common exceptions + "BaseException": BaseException, + "Exception": Exception, + "ArithmeticError": ArithmeticError, + "BufferError": BufferError, + "LookupError": LookupError, + # Adding common modules + "math": __import__("math"), + "itertools": __import__("itertools"), + "functools": __import__("functools"), + "operator": __import__("operator"), + "collections": __import__("collections"), + "re": __import__("re"), + "datetime": __import__("datetime"), +} diff --git a/pyproject.toml b/pyproject.toml index c11605cb6..59970c8ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,8 @@ dependencies = [ "grpcio-health-checking~=1.60.0", "pyyaml", "plotly", - "virtualenv" + "virtualenv", + "restrictedpython==7.1" ] [project.urls] From fdf89fba98a0859216672d9d0b04269df8168fc6 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Fri, 26 Jul 2024 09:32:21 +0200 Subject: [PATCH 03/21] rebuild proto --- fedn/network/grpc/fedn_pb2.py | 85 ++++++------ fedn/network/grpc/fedn_pb2_grpc.py | 206 ++++++++++++++--------------- 2 files changed, 149 insertions(+), 142 deletions(-) diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index c667e1517..36a975d4a 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: fedn/network/grpc/fedn.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -14,25 +15,25 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"0\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"]\n\x0f\x46unctionRequest\x12\x0c\n\x04task\x18\x01 \x01(\t\x12\x18\n\x0epayload_string\x18\x02 \x01(\tH\x00\x12\x17\n\rpayload_bytes\x18\x03 \x01(\x0cH\x00\x42\t\n\x07payload\"M\n\x10\x46unctionResponse\x12\x17\n\rresult_string\x18\x02 \x01(\tH\x00\x12\x16\n\x0cresult_bytes\x18\x03 \x01(\x0cH\x00\x42\x08\n\x06result*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbc\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x42\n\x13SetFunctionProvider\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2S\n\x0f\x46unctionService\x12@\n\x0f\x45xecuteFunction\x12\x15.fedn.FunctionRequest\x1a\x16.fedn.FunctionResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'network.grpc.fedn_pb2', _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'fedn.network.grpc.fedn_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_STATUSTYPE']._serialized_start=2313 - _globals['_STATUSTYPE']._serialized_end=2445 - _globals['_QUEUE']._serialized_start=2447 - _globals['_QUEUE']._serialized_end=2483 - _globals['_MODELSTATUS']._serialized_start=2485 - _globals['_MODELSTATUS']._serialized_end=2568 - _globals['_ROLE']._serialized_start=2570 - _globals['_ROLE']._serialized_end=2626 - _globals['_COMMAND']._serialized_start=2628 - _globals['_COMMAND']._serialized_end=2702 - _globals['_CONNECTIONSTATUS']._serialized_start=2704 - _globals['_CONNECTIONSTATUS']._serialized_end=2777 + _globals['_STATUSTYPE']._serialized_start=2506 + _globals['_STATUSTYPE']._serialized_end=2638 + _globals['_QUEUE']._serialized_start=2640 + _globals['_QUEUE']._serialized_end=2676 + _globals['_MODELSTATUS']._serialized_start=2678 + _globals['_MODELSTATUS']._serialized_end=2761 + _globals['_ROLE']._serialized_start=2763 + _globals['_ROLE']._serialized_end=2819 + _globals['_COMMAND']._serialized_start=2821 + _globals['_COMMAND']._serialized_end=2895 + _globals['_CONNECTIONSTATUS']._serialized_start=2897 + _globals['_CONNECTIONSTATUS']._serialized_end=2970 _globals['_RESPONSE']._serialized_start=71 _globals['_RESPONSE']._serialized_end=129 _globals['_STATUS']._serialized_start=132 @@ -62,29 +63,35 @@ _globals['_CLIENTLIST']._serialized_start=1724 _globals['_CLIENTLIST']._serialized_end=1766 _globals['_CLIENT']._serialized_start=1768 - _globals['_CLIENT']._serialized_end=1816 - _globals['_REASSIGNREQUEST']._serialized_start=1818 - _globals['_REASSIGNREQUEST']._serialized_end=1927 - _globals['_RECONNECTREQUEST']._serialized_start=1929 - _globals['_RECONNECTREQUEST']._serialized_end=2028 - _globals['_PARAMETER']._serialized_start=2030 - _globals['_PARAMETER']._serialized_end=2069 - _globals['_CONTROLREQUEST']._serialized_start=2071 - _globals['_CONTROLREQUEST']._serialized_end=2155 - _globals['_CONTROLRESPONSE']._serialized_start=2157 - _globals['_CONTROLRESPONSE']._serialized_end=2227 - _globals['_CONNECTIONREQUEST']._serialized_start=2229 - _globals['_CONNECTIONREQUEST']._serialized_end=2248 - _globals['_CONNECTIONRESPONSE']._serialized_start=2250 - _globals['_CONNECTIONRESPONSE']._serialized_end=2310 - _globals['_MODELSERVICE']._serialized_start=2779 - _globals['_MODELSERVICE']._serialized_end=2901 - _globals['_CONTROL']._serialized_start=2904 - _globals['_CONTROL']._serialized_end=3152 - _globals['_REDUCER']._serialized_start=3154 - _globals['_REDUCER']._serialized_end=3240 - _globals['_CONNECTOR']._serialized_start=3243 - _globals['_CONNECTOR']._serialized_end=3670 - _globals['_COMBINER']._serialized_start=3673 - _globals['_COMBINER']._serialized_end=3864 + _globals['_CLIENT']._serialized_end=1835 + _globals['_REASSIGNREQUEST']._serialized_start=1837 + _globals['_REASSIGNREQUEST']._serialized_end=1946 + _globals['_RECONNECTREQUEST']._serialized_start=1948 + _globals['_RECONNECTREQUEST']._serialized_end=2047 + _globals['_PARAMETER']._serialized_start=2049 + _globals['_PARAMETER']._serialized_end=2088 + _globals['_CONTROLREQUEST']._serialized_start=2090 + _globals['_CONTROLREQUEST']._serialized_end=2174 + _globals['_CONTROLRESPONSE']._serialized_start=2176 + _globals['_CONTROLRESPONSE']._serialized_end=2246 + _globals['_CONNECTIONREQUEST']._serialized_start=2248 + _globals['_CONNECTIONREQUEST']._serialized_end=2267 + _globals['_CONNECTIONRESPONSE']._serialized_start=2269 + _globals['_CONNECTIONRESPONSE']._serialized_end=2329 + _globals['_FUNCTIONREQUEST']._serialized_start=2331 + _globals['_FUNCTIONREQUEST']._serialized_end=2424 + _globals['_FUNCTIONRESPONSE']._serialized_start=2426 + _globals['_FUNCTIONRESPONSE']._serialized_end=2503 + _globals['_MODELSERVICE']._serialized_start=2972 + _globals['_MODELSERVICE']._serialized_end=3094 + _globals['_CONTROL']._serialized_start=3097 + _globals['_CONTROL']._serialized_end=3413 + _globals['_REDUCER']._serialized_start=3415 + _globals['_REDUCER']._serialized_end=3501 + _globals['_CONNECTOR']._serialized_start=3504 + _globals['_CONNECTOR']._serialized_end=3931 + _globals['_COMBINER']._serialized_start=3934 + _globals['_COMBINER']._serialized_end=4125 + _globals['_FUNCTIONSERVICE']._serialized_start=4127 + _globals['_FUNCTIONSERVICE']._serialized_end=4210 # @@protoc_insertion_point(module_scope) diff --git a/fedn/network/grpc/fedn_pb2_grpc.py b/fedn/network/grpc/fedn_pb2_grpc.py index a1f03ea50..63bf1f625 100644 --- a/fedn/network/grpc/fedn_pb2_grpc.py +++ b/fedn/network/grpc/fedn_pb2_grpc.py @@ -2,7 +2,7 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -from ..grpc import fedn_pb2 as network_dot_grpc_dot_fedn__pb2 +from fedn.network.grpc import fedn_pb2 as fedn_dot_network_dot_grpc_dot_fedn__pb2 class ModelServiceStub(object): @@ -16,13 +16,13 @@ def __init__(self, channel): """ self.Upload = channel.stream_unary( '/fedn.ModelService/Upload', - request_serializer=network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, ) self.Download = channel.unary_stream( '/fedn.ModelService/Download', - request_serializer=network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, ) @@ -46,13 +46,13 @@ def add_ModelServiceServicer_to_server(servicer, server): rpc_method_handlers = { 'Upload': grpc.stream_unary_rpc_method_handler( servicer.Upload, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ModelResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelResponse.SerializeToString, ), 'Download': grpc.unary_stream_rpc_method_handler( servicer.Download, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ModelResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -76,8 +76,8 @@ def Upload(request_iterator, timeout=None, metadata=None): return grpc.experimental.stream_unary(request_iterator, target, '/fedn.ModelService/Upload', - network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -93,8 +93,8 @@ def Download(request, timeout=None, metadata=None): return grpc.experimental.unary_stream(request, target, '/fedn.ModelService/Download', - network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -110,23 +110,23 @@ def __init__(self, channel): """ self.Start = channel.unary_unary( '/fedn.Control/Start', - request_serializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, ) self.Stop = channel.unary_unary( '/fedn.Control/Stop', - request_serializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, ) self.FlushAggregationQueue = channel.unary_unary( '/fedn.Control/FlushAggregationQueue', - request_serializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, ) self.SetAggregator = channel.unary_unary( '/fedn.Control/SetAggregator', - request_serializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, ) self.SetFunctionProvider = channel.unary_unary( '/fedn.Control/SetFunctionProvider', @@ -173,23 +173,23 @@ def add_ControlServicer_to_server(servicer, server): rpc_method_handlers = { 'Start': grpc.unary_unary_rpc_method_handler( servicer.Start, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, ), 'Stop': grpc.unary_unary_rpc_method_handler( servicer.Stop, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, ), 'FlushAggregationQueue': grpc.unary_unary_rpc_method_handler( servicer.FlushAggregationQueue, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, ), 'SetAggregator': grpc.unary_unary_rpc_method_handler( servicer.SetAggregator, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, ), 'SetFunctionProvider': grpc.unary_unary_rpc_method_handler( servicer.SetFunctionProvider, @@ -218,8 +218,8 @@ def Start(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Control/Start', - network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -235,8 +235,8 @@ def Stop(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Control/Stop', - network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -252,8 +252,8 @@ def FlushAggregationQueue(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Control/FlushAggregationQueue', - network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -269,8 +269,8 @@ def SetAggregator(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Control/SetAggregator', - network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -303,8 +303,8 @@ def __init__(self, channel): """ self.GetGlobalModel = channel.unary_unary( '/fedn.Reducer/GetGlobalModel', - request_serializer=network_dot_grpc_dot_fedn__pb2.GetGlobalModelRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.GetGlobalModelResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.GetGlobalModelRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.GetGlobalModelResponse.FromString, ) @@ -322,8 +322,8 @@ def add_ReducerServicer_to_server(servicer, server): rpc_method_handlers = { 'GetGlobalModel': grpc.unary_unary_rpc_method_handler( servicer.GetGlobalModel, - request_deserializer=network_dot_grpc_dot_fedn__pb2.GetGlobalModelRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.GetGlobalModelResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.GetGlobalModelRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.GetGlobalModelResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -347,8 +347,8 @@ def GetGlobalModel(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Reducer/GetGlobalModel', - network_dot_grpc_dot_fedn__pb2.GetGlobalModelRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.GetGlobalModelResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.GetGlobalModelRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.GetGlobalModelResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -364,38 +364,38 @@ def __init__(self, channel): """ self.AllianceStatusStream = channel.unary_stream( '/fedn.Connector/AllianceStatusStream', - request_serializer=network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Status.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Status.FromString, ) self.SendStatus = channel.unary_unary( '/fedn.Connector/SendStatus', - request_serializer=network_dot_grpc_dot_fedn__pb2.Status.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Status.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, ) self.ListActiveClients = channel.unary_unary( '/fedn.Connector/ListActiveClients', - request_serializer=network_dot_grpc_dot_fedn__pb2.ListClientsRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ClientList.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ListClientsRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientList.FromString, ) self.AcceptingClients = channel.unary_unary( '/fedn.Connector/AcceptingClients', - request_serializer=network_dot_grpc_dot_fedn__pb2.ConnectionRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.ConnectionResponse.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ConnectionRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ConnectionResponse.FromString, ) self.SendHeartbeat = channel.unary_unary( '/fedn.Connector/SendHeartbeat', - request_serializer=network_dot_grpc_dot_fedn__pb2.Heartbeat.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Heartbeat.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, ) self.ReassignClient = channel.unary_unary( '/fedn.Connector/ReassignClient', - request_serializer=network_dot_grpc_dot_fedn__pb2.ReassignRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ReassignRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, ) self.ReconnectClient = channel.unary_unary( '/fedn.Connector/ReconnectClient', - request_serializer=network_dot_grpc_dot_fedn__pb2.ReconnectRequest.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ReconnectRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, ) @@ -454,38 +454,38 @@ def add_ConnectorServicer_to_server(servicer, server): rpc_method_handlers = { 'AllianceStatusStream': grpc.unary_stream_rpc_method_handler( servicer.AllianceStatusStream, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Status.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Status.SerializeToString, ), 'SendStatus': grpc.unary_unary_rpc_method_handler( servicer.SendStatus, - request_deserializer=network_dot_grpc_dot_fedn__pb2.Status.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Status.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), 'ListActiveClients': grpc.unary_unary_rpc_method_handler( servicer.ListActiveClients, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ListClientsRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ClientList.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ListClientsRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientList.SerializeToString, ), 'AcceptingClients': grpc.unary_unary_rpc_method_handler( servicer.AcceptingClients, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ConnectionRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.ConnectionResponse.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ConnectionRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ConnectionResponse.SerializeToString, ), 'SendHeartbeat': grpc.unary_unary_rpc_method_handler( servicer.SendHeartbeat, - request_deserializer=network_dot_grpc_dot_fedn__pb2.Heartbeat.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Heartbeat.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), 'ReassignClient': grpc.unary_unary_rpc_method_handler( servicer.ReassignClient, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ReassignRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ReassignRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), 'ReconnectClient': grpc.unary_unary_rpc_method_handler( servicer.ReconnectClient, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ReconnectRequest.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ReconnectRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -509,8 +509,8 @@ def AllianceStatusStream(request, timeout=None, metadata=None): return grpc.experimental.unary_stream(request, target, '/fedn.Connector/AllianceStatusStream', - network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Status.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Status.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -526,8 +526,8 @@ def SendStatus(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Connector/SendStatus', - network_dot_grpc_dot_fedn__pb2.Status.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Response.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Status.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -543,8 +543,8 @@ def ListActiveClients(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Connector/ListActiveClients', - network_dot_grpc_dot_fedn__pb2.ListClientsRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ClientList.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ListClientsRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientList.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -560,8 +560,8 @@ def AcceptingClients(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Connector/AcceptingClients', - network_dot_grpc_dot_fedn__pb2.ConnectionRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.ConnectionResponse.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ConnectionRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ConnectionResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -577,8 +577,8 @@ def SendHeartbeat(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Connector/SendHeartbeat', - network_dot_grpc_dot_fedn__pb2.Heartbeat.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Response.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Heartbeat.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -594,8 +594,8 @@ def ReassignClient(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Connector/ReassignClient', - network_dot_grpc_dot_fedn__pb2.ReassignRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Response.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ReassignRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -611,8 +611,8 @@ def ReconnectClient(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Connector/ReconnectClient', - network_dot_grpc_dot_fedn__pb2.ReconnectRequest.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Response.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ReconnectRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -628,18 +628,18 @@ def __init__(self, channel): """ self.TaskStream = channel.unary_stream( '/fedn.Combiner/TaskStream', - request_serializer=network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.TaskRequest.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.TaskRequest.FromString, ) self.SendModelUpdate = channel.unary_unary( '/fedn.Combiner/SendModelUpdate', - request_serializer=network_dot_grpc_dot_fedn__pb2.ModelUpdate.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelUpdate.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, ) self.SendModelValidation = channel.unary_unary( '/fedn.Combiner/SendModelValidation', - request_serializer=network_dot_grpc_dot_fedn__pb2.ModelValidation.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelValidation.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, ) @@ -670,18 +670,18 @@ def add_CombinerServicer_to_server(servicer, server): rpc_method_handlers = { 'TaskStream': grpc.unary_stream_rpc_method_handler( servicer.TaskStream, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.TaskRequest.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.TaskRequest.SerializeToString, ), 'SendModelUpdate': grpc.unary_unary_rpc_method_handler( servicer.SendModelUpdate, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelUpdate.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelUpdate.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), 'SendModelValidation': grpc.unary_unary_rpc_method_handler( servicer.SendModelValidation, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelValidation.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelValidation.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -705,8 +705,8 @@ def TaskStream(request, timeout=None, metadata=None): return grpc.experimental.unary_stream(request, target, '/fedn.Combiner/TaskStream', - network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, - network_dot_grpc_dot_fedn__pb2.TaskRequest.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientAvailableMessage.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.TaskRequest.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -722,8 +722,8 @@ def SendModelUpdate(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Combiner/SendModelUpdate', - network_dot_grpc_dot_fedn__pb2.ModelUpdate.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Response.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelUpdate.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -739,8 +739,8 @@ def SendModelValidation(request, timeout=None, metadata=None): return grpc.experimental.unary_unary(request, target, '/fedn.Combiner/SendModelValidation', - network_dot_grpc_dot_fedn__pb2.ModelValidation.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Response.FromString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelValidation.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From 1f58028b9b6c98213ea62cbc3e92179fcc66f0fe Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Fri, 26 Jul 2024 09:36:28 +0200 Subject: [PATCH 04/21] linter fix --- fedn/network/combiner/aggregators/custom.py | 2 -- fedn/network/hooks/hooks.py | 1 - 2 files changed, 3 deletions(-) diff --git a/fedn/network/combiner/aggregators/custom.py b/fedn/network/combiner/aggregators/custom.py index ab0270c67..d50fbe37a 100644 --- a/fedn/network/combiner/aggregators/custom.py +++ b/fedn/network/combiner/aggregators/custom.py @@ -1,7 +1,5 @@ import traceback -import numpy as np - from fedn.common.log_config import logger from fedn.network.combiner.aggregators.aggregatorbase import AggregatorBase diff --git a/fedn/network/hooks/hooks.py b/fedn/network/hooks/hooks.py index 290720d96..cced063c2 100644 --- a/fedn/network/hooks/hooks.py +++ b/fedn/network/hooks/hooks.py @@ -1,5 +1,4 @@ from concurrent import futures -from io import BytesIO import grpc From 76b59497505cdbf52c458e1c52b324641268fd5d Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Mon, 2 Sep 2024 09:38:56 +0200 Subject: [PATCH 05/21] add custom aggregator and metadata --- examples/custom-aggregator/.dockerignore | 4 + examples/custom-aggregator/.gitignore | 6 + examples/custom-aggregator/README.rst | 8 ++ .../custom-aggregator/client/aggregator.py | 113 +++++++++++++++++ examples/custom-aggregator/client/data.py | 97 ++++++++++++++ examples/custom-aggregator/client/fedn.yaml | 12 ++ examples/custom-aggregator/client/model.py | 76 +++++++++++ examples/custom-aggregator/client/predict.py | 37 ++++++ .../custom-aggregator/client/python_env.yaml | 9 ++ examples/custom-aggregator/client/train.py | 118 ++++++++++++++++++ examples/custom-aggregator/client/validate.py | 55 ++++++++ .../docker-compose.override.yaml | 35 ++++++ examples/mnist-pytorch/client/aggregator.py | 21 ---- fedn/cli/hooks_cmd.py | 2 +- fedn/network/clients/client.py | 12 +- .../combiner/aggregators/aggregatorbase.py | 4 + fedn/network/combiner/aggregators/custom.py | 11 +- .../aggregators/functionproviderbase.py | 63 ++++++++++ fedn/network/combiner/hook_client.py | 21 +++- fedn/network/{ => combiner}/hooks/__init__.py | 0 fedn/network/{ => combiner}/hooks/hooks.py | 32 +++-- .../{ => combiner}/hooks/safe_builtins.py | 2 - fedn/network/combiner/roundhandler.py | 1 + fedn/network/controller/control.py | 2 + fedn/network/grpc/fedn.proto | 6 +- fedn/network/grpc/fedn_pb2.py | 56 ++++----- fedn/utils/helpers/helpers.py | 13 ++ 27 files changed, 738 insertions(+), 78 deletions(-) create mode 100644 examples/custom-aggregator/.dockerignore create mode 100644 examples/custom-aggregator/.gitignore create mode 100644 examples/custom-aggregator/README.rst create mode 100644 examples/custom-aggregator/client/aggregator.py create mode 100644 examples/custom-aggregator/client/data.py create mode 100644 examples/custom-aggregator/client/fedn.yaml create mode 100644 examples/custom-aggregator/client/model.py create mode 100644 examples/custom-aggregator/client/predict.py create mode 100644 examples/custom-aggregator/client/python_env.yaml create mode 100644 examples/custom-aggregator/client/train.py create mode 100644 examples/custom-aggregator/client/validate.py create mode 100644 examples/custom-aggregator/docker-compose.override.yaml delete mode 100644 examples/mnist-pytorch/client/aggregator.py create mode 100644 fedn/network/combiner/aggregators/functionproviderbase.py rename fedn/network/{ => combiner}/hooks/__init__.py (100%) rename fedn/network/{ => combiner}/hooks/hooks.py (67%) rename fedn/network/{ => combiner}/hooks/safe_builtins.py (96%) diff --git a/examples/custom-aggregator/.dockerignore b/examples/custom-aggregator/.dockerignore new file mode 100644 index 000000000..8ba9024ad --- /dev/null +++ b/examples/custom-aggregator/.dockerignore @@ -0,0 +1,4 @@ +data +seed.npz +*.tgz +*.tar.gz \ No newline at end of file diff --git a/examples/custom-aggregator/.gitignore b/examples/custom-aggregator/.gitignore new file mode 100644 index 000000000..a9f01054b --- /dev/null +++ b/examples/custom-aggregator/.gitignore @@ -0,0 +1,6 @@ +data +*.npz +*.tgz +*.tar.gz +.mnist-pytorch +client.yaml \ No newline at end of file diff --git a/examples/custom-aggregator/README.rst b/examples/custom-aggregator/README.rst new file mode 100644 index 000000000..33ffbaa06 --- /dev/null +++ b/examples/custom-aggregator/README.rst @@ -0,0 +1,8 @@ +FEDn Project: Custom aggregator (hyperparameter tuning) +----------------------------- + +Will be updated after studio update. + +To run custom aggregators: + +client.start_session(aggregator="custom", function_provider_path="client/aggregator.py", rounds=101) diff --git a/examples/custom-aggregator/client/aggregator.py b/examples/custom-aggregator/client/aggregator.py new file mode 100644 index 000000000..a4816623b --- /dev/null +++ b/examples/custom-aggregator/client/aggregator.py @@ -0,0 +1,113 @@ +import numpy as np + +from fedn.common.log_config import logger +from fedn.network.combiner.aggregators.functionproviderbase import FunctionProviderBase + + +class FunctionProvider(FunctionProviderBase): + """A FunctionProvider class responsible for aggregating model parameters + from multiple clients and performing hyperparameter tuning by adjusting + the learning rate every 20th round. The class logs the current state of + the model, learning rate, and round to facilitate monitoring and evaluation. + """ + + def __init__(self) -> None: + self.current_round = -1 + self.initial_parameters = None + self.learning_rates = [0.001, 0.01, 0.0001, 0.1, 0.00001] + self.current_lr_index = -1 + self.current_lr = 0 # start with 0 learning rate the first round to get initial parameters + self.current_parameters = None + + # Tracking metrics + self.highest_accuracy = 0 + self.highest_accuracy_round = -1 + self.highest_accuracy_lr = 0 + self.mean_loss_per_lr = [] + self.mean_acc_per_lr = [] + self.highest_mean_acc = 0 + self.highest_mean_acc_round = -1 + self.highest_mean_acc_lr = None + + def aggregate(self, results: list[tuple[list[np.ndarray], dict]]) -> list[np.ndarray]: + """Aggregate model parameters using weighted average based on the number of examples each client has. + + Args: + ---- + results (list of tuples): Each tuple contains: + - A list of numpy.ndarrays representing model parameters from a client. + - A dictionary containing client metadata, which must include a key "num_examples" indicating + the number of examples used by the client. + + Returns: + ------- + list of numpy.ndarrays: Aggregated model parameters as a list of numpy.ndarrays. + + """ + total_loss = 0 + total_acc = 0 + num_clients = len(results) + if self.current_round == -1: + self.initial_parameters = results[0][0] + averaged_parameters = self.initial_parameters # first round no updates were made. + elif self.current_round % 20 == 0: + if self.mean_loss_per_lr: + logger.info(f"Completed Learning Rate: {self.current_lr}") + logger.info(f"Mean Loss: {np.mean(self.mean_loss_per_lr)}, Highest Accuracy: {np.max(self.mean_acc_per_lr)}") + logger.info( + f"""Highest mean accuracy across rounds: {self.highest_mean_acc} + at round {self.highest_mean_acc_round} with lr {self.highest_mean_acc_lr}""" + ) + + # Reset tracking for the new learning rate + self.mean_loss_per_lr = [] + self.mean_acc_per_lr = [] + + averaged_parameters = self.initial_parameters + self.current_lr_index += 1 + self.current_lr = self.learning_rates[self.current_lr_index] + else: + # Aggregate using fedavg + summed_parameters = [np.zeros_like(param) for param in results[0][0]] + total_weight = 0 + for client_params, client_metadata in results: + weight = client_metadata.get("num_examples", 1) + total_weight += weight + for i, param in enumerate(client_params): + summed_parameters[i] += param * weight + + total_loss += client_metadata.get("test_loss", 0) + total_acc += client_metadata.get("test_acc", 0) + + averaged_parameters = [param / total_weight for param in summed_parameters] + + # Calculate average loss and accuracy by number of clients + avg_loss = total_loss / num_clients if num_clients > 0 else 0 + avg_acc = total_acc / num_clients if num_clients > 0 else 0 + + # Update the tracking for the current learning rate + self.mean_loss_per_lr.append(avg_loss) + self.mean_acc_per_lr.append(avg_acc) + + # Check if we have a new highest accuracy + if avg_acc > self.highest_accuracy: + self.highest_accuracy = avg_acc + self.highest_accuracy_round = self.current_round + self.highest_accuracy_lr = self.current_lr + + # Check if we have a new highest mean accuracy across rounds + if avg_acc > self.highest_mean_acc: + self.highest_mean_acc = avg_acc + self.highest_mean_acc_round = self.current_round + self.highest_mean_acc_lr = self.current_lr + + # Print the metrics + logger.info(f"Round {self.current_round} - Learning Rate: {self.current_lr}") + logger.info(f"Average Test Loss: {avg_loss}, Average Test Accuracy: {avg_acc}") + logger.info(f"Highest Accuracy Achieved: {self.highest_accuracy} at round {self.highest_accuracy_round} with lr {self.highest_accuracy_lr}") + + self.current_round += 1 + return averaged_parameters + + def get_model_metadata(self): + return {"learning_rate": self.current_lr, "parameter_tuning": True} diff --git a/examples/custom-aggregator/client/data.py b/examples/custom-aggregator/client/data.py new file mode 100644 index 000000000..b921f3132 --- /dev/null +++ b/examples/custom-aggregator/client/data.py @@ -0,0 +1,97 @@ +import os +from math import floor + +import torch +import torchvision + +dir_path = os.path.dirname(os.path.realpath(__file__)) +abs_path = os.path.abspath(dir_path) + + +def get_data(out_dir="data"): + # Make dir if necessary + if not os.path.exists(out_dir): + os.mkdir(out_dir) + + # Only download if not already downloaded + if not os.path.exists(f"{out_dir}/train"): + torchvision.datasets.MNIST(root=f"{out_dir}/train", transform=torchvision.transforms.ToTensor, train=True, download=True) + if not os.path.exists(f"{out_dir}/test"): + torchvision.datasets.MNIST(root=f"{out_dir}/test", transform=torchvision.transforms.ToTensor, train=False, download=True) + + +def load_data(data_path, is_train=True): + """Load data from disk. + + :param data_path: Path to data file. + :type data_path: str + :param is_train: Whether to load training or test data. + :type is_train: bool + :return: Tuple of data and labels. + :rtype: tuple + """ + if data_path is None: + data_path = os.environ.get("FEDN_DATA_PATH", abs_path + "/data/clients/1/mnist.pt") + + data = torch.load(data_path) + + if is_train: + X = data["x_train"] + y = data["y_train"] + else: + X = data["x_test"] + y = data["y_test"] + + # Normalize + X = X / 255 + + return X, y + + +def splitset(dataset, parts): + n = dataset.shape[0] + local_n = floor(n / parts) + result = [] + for i in range(parts): + result.append(dataset[i * local_n : (i + 1) * local_n]) + return result + + +def split(out_dir="data"): + n_splits = int(os.environ.get("FEDN_NUM_DATA_SPLITS", 2)) + + # Make dir + if not os.path.exists(f"{out_dir}/clients"): + os.mkdir(f"{out_dir}/clients") + + # Load and convert to dict + train_data = torchvision.datasets.MNIST(root=f"{out_dir}/train", transform=torchvision.transforms.ToTensor, train=True) + test_data = torchvision.datasets.MNIST(root=f"{out_dir}/test", transform=torchvision.transforms.ToTensor, train=False) + data = { + "x_train": splitset(train_data.data, n_splits), + "y_train": splitset(train_data.targets, n_splits), + "x_test": splitset(test_data.data, n_splits), + "y_test": splitset(test_data.targets, n_splits), + } + + # Make splits + for i in range(n_splits): + subdir = f"{out_dir}/clients/{str(i+1)}" + if not os.path.exists(subdir): + os.mkdir(subdir) + torch.save( + { + "x_train": data["x_train"][i], + "y_train": data["y_train"][i], + "x_test": data["x_test"][i], + "y_test": data["y_test"][i], + }, + f"{subdir}/mnist.pt", + ) + + +if __name__ == "__main__": + # Prepare data if not already done + if not os.path.exists(abs_path + "/data/clients/1"): + get_data() + split() diff --git a/examples/custom-aggregator/client/fedn.yaml b/examples/custom-aggregator/client/fedn.yaml new file mode 100644 index 000000000..30873488b --- /dev/null +++ b/examples/custom-aggregator/client/fedn.yaml @@ -0,0 +1,12 @@ +python_env: python_env.yaml +entry_points: + build: + command: python model.py + startup: + command: python data.py + train: + command: python train.py + validate: + command: python validate.py + predict: + command: python predict.py \ No newline at end of file diff --git a/examples/custom-aggregator/client/model.py b/examples/custom-aggregator/client/model.py new file mode 100644 index 000000000..6ad344770 --- /dev/null +++ b/examples/custom-aggregator/client/model.py @@ -0,0 +1,76 @@ +import collections + +import torch + +from fedn.utils.helpers.helpers import get_helper + +HELPER_MODULE = "numpyhelper" +helper = get_helper(HELPER_MODULE) + + +def compile_model(): + """Compile the pytorch model. + + :return: The compiled model. + :rtype: torch.nn.Module + """ + + class Net(torch.nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = torch.nn.Linear(784, 64) + self.fc2 = torch.nn.Linear(64, 32) + self.fc3 = torch.nn.Linear(32, 10) + + def forward(self, x): + x = torch.nn.functional.relu(self.fc1(x.reshape(x.size(0), 784))) + x = torch.nn.functional.dropout(x, p=0.5, training=self.training) + x = torch.nn.functional.relu(self.fc2(x)) + x = torch.nn.functional.log_softmax(self.fc3(x), dim=1) + return x + + return Net() + + +def save_parameters(model, out_path): + """Save model paramters to file. + + :param model: The model to serialize. + :type model: torch.nn.Module + :param out_path: The path to save to. + :type out_path: str + """ + parameters_np = [val.cpu().numpy() for _, val in model.state_dict().items()] + helper.save(parameters_np, out_path) + + +def load_parameters(model_path): + """Load model parameters from file and populate model. + + param model_path: The path to load from. + :type model_path: str + :return: The loaded model. + :rtype: torch.nn.Module + """ + model = compile_model() + parameters_np = helper.load(model_path) + + params_dict = zip(model.state_dict().keys(), parameters_np) + state_dict = collections.OrderedDict({key: torch.tensor(x) for key, x in params_dict}) + model.load_state_dict(state_dict, strict=True) + return model + + +def init_seed(out_path="seed.npz"): + """Initialize seed model and save it to file. + + :param out_path: The path to save the seed model to. + :type out_path: str + """ + # Init and save + model = compile_model() + save_parameters(model, out_path) + + +if __name__ == "__main__": + init_seed("../seed.npz") diff --git a/examples/custom-aggregator/client/predict.py b/examples/custom-aggregator/client/predict.py new file mode 100644 index 000000000..aaf9f0f50 --- /dev/null +++ b/examples/custom-aggregator/client/predict.py @@ -0,0 +1,37 @@ +import os +import sys + +import torch +from data import load_data +from model import load_parameters + +dir_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.abspath(dir_path)) + + +def predict(in_model_path, out_artifact_path, data_path=None): + """Validate model. + + :param in_model_path: The path to the input model. + :type in_model_path: str + :param out_artifact_path: The path to save the predict output to. + :type out_artifact_path: str + :param data_path: The path to the data file. + :type data_path: str + """ + # Load data + x_test, y_test = load_data(data_path, is_train=False) + + # Load model + model = load_parameters(in_model_path) + model.eval() + + # Predict + with torch.no_grad(): + y_pred = model(x_test) + # Save prediction to file/artifact, the artifact will be uploaded to the object store by the client + torch.save(y_pred, out_artifact_path) + + +if __name__ == "__main__": + predict(sys.argv[1], sys.argv[2]) diff --git a/examples/custom-aggregator/client/python_env.yaml b/examples/custom-aggregator/client/python_env.yaml new file mode 100644 index 000000000..afdea926f --- /dev/null +++ b/examples/custom-aggregator/client/python_env.yaml @@ -0,0 +1,9 @@ +name: mnist-pytorch +build_dependencies: + - pip + - setuptools + - wheel +dependencies: + - torch==2.3.1 + - torchvision==0.18.1 + - fedn diff --git a/examples/custom-aggregator/client/train.py b/examples/custom-aggregator/client/train.py new file mode 100644 index 000000000..18b61008b --- /dev/null +++ b/examples/custom-aggregator/client/train.py @@ -0,0 +1,118 @@ +import json +import math +import os +import sys + +import torch +from model import load_parameters, save_parameters + +from data import load_data +from fedn.utils.helpers.helpers import save_metadata + + +# swap this to the load_metadata from helpers.helpers on release.. +def load_metadata(filename): + """Load metadata from file. + + :param filename: The name of the file to load from. + :type filename: str + :return: The loaded metadata. + :rtype: dict + """ + with open(filename + "-metadata", "r") as infile: + metadata = json.load(infile) + return metadata + + +dir_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.abspath(dir_path)) + + +def validate(model, data_path): + """Validate a model.""" + x_test, y_test = load_data(data_path, is_train=False) + model.eval() + + model.train() + criterion = torch.nn.NLLLoss() + with torch.no_grad(): + test_out = model(x_test) + test_loss = criterion(test_out, y_test) + test_accuracy = torch.sum(torch.argmax(test_out, dim=1) == y_test) / len(test_out) + return test_loss, test_accuracy + + +def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1, lr=0.01): + """Complete a model update. + + Load model paramters from in_model_path (managed by the FEDn client), + perform a model update, and write updated paramters + to out_model_path (picked up by the FEDn client). + + :param in_model_path: The path to the input model. + :type in_model_path: str + :param out_model_path: The path to save the output model to. + :type out_model_path: str + :param data_path: The path to the data file. + :type data_path: str + :param batch_size: The batch size to use. + :type batch_size: int + :param epochs: The number of epochs to train. + :type epochs: int + :param lr: The learning rate to use. + :type lr: float + """ + # Load data + x_train, y_train = load_data(data_path) + + # Load parmeters and initialize model + model = load_parameters(in_model_path) + + model_metadata = load_metadata(in_model_path) + lr = model_metadata["learning_rate"] + + # validate each new aggregated model for hyperparameter tuning + if model_metadata["parameter_tuning"]: + test_loss, test_acc = validate(model, data_path) + else: + test_loss, test_acc = torch.tensor(0.0), torch.tensor(0.0) + + # Train + optimizer = torch.optim.SGD(model.parameters(), lr=lr) + n_batches = int(math.ceil(len(x_train) / batch_size)) + criterion = torch.nn.NLLLoss() + for e in range(epochs): # epoch loop + for b in range(n_batches): # batch loop + # Retrieve current batch + batch_x = x_train[b * batch_size : (b + 1) * batch_size] + batch_y = y_train[b * batch_size : (b + 1) * batch_size] + # Train on batch + optimizer.zero_grad() + outputs = model(batch_x) + loss = criterion(outputs, batch_y) + loss.backward() + optimizer.step() + # Log + if b % 100 == 0: + print(f"Epoch {e}/{epochs-1} | Batch: {b}/{n_batches-1} | Loss: {loss.item()}") + + # Metadata needed for aggregation server side + metadata = { + # num_examples are mandatory + "num_examples": len(x_train), + "batch_size": batch_size, + "epochs": epochs, + "lr": lr, + "test_loss": test_loss.item(), + "test_acc": test_acc.item(), + } + + # Save JSON metadata file (mandatory) + save_metadata(metadata, out_model_path) + + # Save model update (mandatory) + save_parameters(model, out_model_path) + + +if __name__ == "__main__": + train(sys.argv[1], sys.argv[2]) diff --git a/examples/custom-aggregator/client/validate.py b/examples/custom-aggregator/client/validate.py new file mode 100644 index 000000000..09328181f --- /dev/null +++ b/examples/custom-aggregator/client/validate.py @@ -0,0 +1,55 @@ +import os +import sys + +import torch +from model import load_parameters + +from data import load_data +from fedn.utils.helpers.helpers import save_metrics + +dir_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.abspath(dir_path)) + + +def validate(in_model_path, out_json_path, data_path=None): + """Validate model. + + :param in_model_path: The path to the input model. + :type in_model_path: str + :param out_json_path: The path to save the output JSON to. + :type out_json_path: str + :param data_path: The path to the data file. + :type data_path: str + """ + # Load data + x_train, y_train = load_data(data_path) + x_test, y_test = load_data(data_path, is_train=False) + + # Load model + model = load_parameters(in_model_path) + model.eval() + + # Evaluate + criterion = torch.nn.NLLLoss() + with torch.no_grad(): + train_out = model(x_train) + training_loss = criterion(train_out, y_train) + training_accuracy = torch.sum(torch.argmax(train_out, dim=1) == y_train) / len(train_out) + test_out = model(x_test) + test_loss = criterion(test_out, y_test) + test_accuracy = torch.sum(torch.argmax(test_out, dim=1) == y_test) / len(test_out) + + # JSON schema + report = { + "training_loss": training_loss.item(), + "training_accuracy": training_accuracy.item(), + "test_loss": test_loss.item(), + "test_accuracy": test_accuracy.item(), + } + + # Save JSON + save_metrics(report, out_json_path) + + +if __name__ == "__main__": + validate(sys.argv[1], sys.argv[2]) diff --git a/examples/custom-aggregator/docker-compose.override.yaml b/examples/custom-aggregator/docker-compose.override.yaml new file mode 100644 index 000000000..822a696dc --- /dev/null +++ b/examples/custom-aggregator/docker-compose.override.yaml @@ -0,0 +1,35 @@ +# Compose schema version +version: '3.4' + +# Overriding requirements + +x-env: &defaults + GET_HOSTS_FROM: dns + FEDN_PACKAGE_EXTRACT_DIR: package + FEDN_NUM_DATA_SPLITS: 2 + +services: + + client1: + extends: + file: ${HOST_REPO_DIR:-.}/docker-compose.yaml + service: client + environment: + <<: *defaults + FEDN_DATA_PATH: /app/package/client/data/clients/1/mnist.pt + deploy: + replicas: 1 + volumes: + - ${HOST_REPO_DIR:-.}/fedn:/app/fedn + + client2: + extends: + file: ${HOST_REPO_DIR:-.}/docker-compose.yaml + service: client + environment: + <<: *defaults + FEDN_DATA_PATH: /app/package/client/data/clients/2/mnist.pt + deploy: + replicas: 1 + volumes: + - ${HOST_REPO_DIR:-.}/fedn:/app/fedn diff --git a/examples/mnist-pytorch/client/aggregator.py b/examples/mnist-pytorch/client/aggregator.py deleted file mode 100644 index 65d829429..000000000 --- a/examples/mnist-pytorch/client/aggregator.py +++ /dev/null @@ -1,21 +0,0 @@ -import numpy as np - - -class FunctionProvider: - def __init__(self) -> None: - pass - - def aggregate(self, parameters): - if len(parameters) == 0: - return [] - num_clients = len(parameters) - - summed_parameters = [np.zeros_like(param) for param in parameters[0]] - - for client_params in parameters: - for i, param in enumerate(client_params): - summed_parameters[i] += param - - averaged_parameters = [param / num_clients for param in summed_parameters] - - return averaged_parameters diff --git a/fedn/cli/hooks_cmd.py b/fedn/cli/hooks_cmd.py index e3515bc5a..1b263fee3 100644 --- a/fedn/cli/hooks_cmd.py +++ b/fedn/cli/hooks_cmd.py @@ -1,6 +1,6 @@ import click -from fedn.network.hooks.hooks import serve +from fedn.network.combiner.hooks.hooks import serve from .main import main diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index e594d7b6d..2cb37bcaf 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -29,7 +29,7 @@ from fedn.network.clients.state import ClientState, ClientStateToString from fedn.network.combiner.modelservice import get_tmp_path, upload_request_generator from fedn.utils.dispatcher import Dispatcher -from fedn.utils.helpers.helpers import get_helper +from fedn.utils.helpers.helpers import get_helper, load_metadata, save_metadata CHUNK_SIZE = 1024 * 1024 VALID_NAME_REGEX = "^[a-zA-Z0-9_-]*$" @@ -484,7 +484,7 @@ def _listen_to_task_stream(self): if not self._connected: return - def _process_training_request(self, model_id: str, session_id: str = None): + def _process_training_request(self, model_id: str, session_id: str = None, model_metadata: dict = None): """Process a training (model update) request. :param model_id: The model id of the model to be updated. @@ -510,6 +510,8 @@ def _process_training_request(self, model_id: str, session_id: str = None): with open(inpath, "wb") as fh: fh.write(mdl.getbuffer()) + save_metadata(metadata=model_metadata, filename=inpath) + outpath = self.helper.get_tmp_path() tic = time.time() # TODO: Check return status, fail gracefully @@ -530,8 +532,7 @@ def _process_training_request(self, model_id: str, session_id: str = None): meta["upload_model"] = time.time() - tic # Read the metadata file - with open(outpath + "-metadata", "r") as fh: - training_metadata = json.loads(fh.read()) + training_metadata = load_metadata(outpath) meta["training_metadata"] = training_metadata os.unlink(inpath) @@ -650,7 +651,8 @@ def process_request(self): if task_type == "train": tic = time.time() self.state = ClientState.training - model_id, meta = self._process_training_request(request.model_id, session_id=request.session_id) + model_metadata = json.loads(request.data)["model_metadata"] + model_id, meta = self._process_training_request(request.model_id, session_id=request.session_id, model_metadata=model_metadata) if meta is not None: processing_time = time.time() - tic diff --git a/fedn/network/combiner/aggregators/aggregatorbase.py b/fedn/network/combiner/aggregators/aggregatorbase.py index 44d10fca2..4014106a1 100644 --- a/fedn/network/combiner/aggregators/aggregatorbase.py +++ b/fedn/network/combiner/aggregators/aggregatorbase.py @@ -55,6 +55,10 @@ def combine_models(self, nr_expected_models=None, nr_required_models=1, helper=N """ pass + def get_model_metadata(self): + """Sends metadata to the clients.""" + return {} + def on_model_update(self, model_update): """Callback when a new client model update is recieved. diff --git a/fedn/network/combiner/aggregators/custom.py b/fedn/network/combiner/aggregators/custom.py index d50fbe37a..c56dc4b73 100644 --- a/fedn/network/combiner/aggregators/custom.py +++ b/fedn/network/combiner/aggregators/custom.py @@ -27,6 +27,13 @@ def __init__(self, storage, server, modelservice, round_handler): self.name = "custom" self.code_set = False + def get_model_metadata(self): + """Sends metadata to the clients.""" + if not self.code_set: + self.round_handler.set_function_provider() + self.code_set = True + return self.round_handler.combiner_hook_client.get_model_metadata() + def combine_models(self, helper=None, delete_models=True, parameters=None): """Aggregate all model updates with custom aggregator. @@ -76,9 +83,9 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): self.model_updates.task_done() if not self.model_updates.empty(): - self.round_handler.combiner_hook_client.call_function("store_parameters", model_next, helper) + self.round_handler.combiner_hook_client.call_function("store_parameters", model_next, helper, metadata) else: - model = self.round_handler.combiner_hook_client.call_function("aggregate", model_next, helper) + model = self.round_handler.combiner_hook_client.call_function("aggregate", model_next, helper, metadata) except Exception as e: tb = traceback.format_exc() logger.error(f"AGGREGATOR({self.name}): Error encoutered while processing model update: {e}") diff --git a/fedn/network/combiner/aggregators/functionproviderbase.py b/fedn/network/combiner/aggregators/functionproviderbase.py new file mode 100644 index 000000000..ea53b7553 --- /dev/null +++ b/fedn/network/combiner/aggregators/functionproviderbase.py @@ -0,0 +1,63 @@ +from abc import ABC, ABCMeta, abstractmethod + +import numpy as np + + +class NamingEnforcementMeta(type): + """Metaclass that enforces the name 'FunctionProvider' for any class + inheriting from FunctionProviderBase. + """ + + def __init__(cls, name, bases, attrs): + if name != "FunctionProvider" and name != "FunctionProviderBase": # Allow the base class name + raise TypeError(f"Class {name} must be named 'FunctionProvider'") + super().__init__(name, bases, attrs) + + +class CombinedMeta(NamingEnforcementMeta, ABCMeta): + """A metaclass that combines ABCMeta and NamingEnforcementMeta to allow + the use of both abstract base classes and custom naming enforcement. + """ + + pass + + +class FunctionProviderBase(ABC, metaclass=CombinedMeta): + """Abstract base class that defines the structure for function providers. + Enforces the implementation of certain methods and provides shared functionality. + """ + + def __init__(self) -> None: + """Initialize the FunctionProviderBase class. This method can be overridden + by subclasses if initialization logic is required. + """ + pass + + @abstractmethod + def aggregate(self, parameters: list[list[np.ndarray]]) -> list[np.ndarray]: + """Aggregates a list of parameters from clients. + + Args: + ---- + parameters (list[list[np.ndarray]]): A list where each element is a list + of numpy arrays representing parameters from a client. + + Returns: + ------- + list[np.ndarray]: A list of numpy arrays representing the aggregated + parameters across all clients. + + """ + pass + + def get_model_metadata(self) -> dict: + """Returns metadata related to the model, which gets distributed to the clients. + The dictionary may only contain primitive types. + + + Returns + ------- + dict: A dictionary containing metadata information. + + """ + return {} diff --git a/fedn/network/combiner/hook_client.py b/fedn/network/combiner/hook_client.py index 422c5cb94..8b8555ee9 100644 --- a/fedn/network/combiner/hook_client.py +++ b/fedn/network/combiner/hook_client.py @@ -1,3 +1,4 @@ +import json import os import grpc @@ -15,8 +16,11 @@ def __init__(self): self.channel = grpc.insecure_channel(self.hook_service_host) self.stub = rpc.FunctionServiceStub(self.channel) - def call_function_service(self, task, payload): - request = fedn.FunctionRequest(task=task, payload_string=payload) if task == "setup" else fedn.FunctionRequest(task=task, payload_bytes=payload) + def call_function_service(self, task, payload, meta={}): + if task == "setup" or task == "get_model_metadata": + request = fedn.FunctionRequest(task=task, payload_string=payload) + if task == "aggregate" or task == "store_parameters": + request = fedn.FunctionRequest(task=task, payload_bytes=payload, payload_string=json.dumps(meta)) try: response = self.stub.ExecuteFunction(request) return response @@ -24,17 +28,22 @@ def call_function_service(self, task, payload): logger.info(f"RPC failed: {e}") return None - # Example method to trigger function execution def set_function_provider(self, class_code): if not isinstance(class_code, str): raise TypeError("class_code must be of type string") self.call_function_service("setup", class_code) + logger.info("Function provider code set.") - def call_function(self, task, payload, helper): + def call_function(self, task, payload, helper, meta): if task == "aggregate": payload = serialize_model_to_BytesIO(payload, helper).getvalue() - response = self.call_function_service(task, payload) + response = self.call_function_service(task, payload, meta=meta) return load_model_from_BytesIO(response.result_bytes, helper) if task == "store_parameters": payload = serialize_model_to_BytesIO(payload, helper).getvalue() - response = self.call_function_service(task, payload) + response = self.call_function_service(task, payload, meta=meta) + + def get_model_metadata(self): + response = self.call_function_service("get_model_metadata", "") + model_metadata_json = response.result_string + return json.loads(model_metadata_json) diff --git a/fedn/network/hooks/__init__.py b/fedn/network/combiner/hooks/__init__.py similarity index 100% rename from fedn/network/hooks/__init__.py rename to fedn/network/combiner/hooks/__init__.py diff --git a/fedn/network/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py similarity index 67% rename from fedn/network/hooks/hooks.py rename to fedn/network/combiner/hooks/hooks.py index cced063c2..55ba08a6e 100644 --- a/fedn/network/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -1,3 +1,4 @@ +import json from concurrent import futures import grpc @@ -5,8 +6,8 @@ import fedn.network.grpc.fedn_pb2 as fedn import fedn.network.grpc.fedn_pb2_grpc as rpc from fedn.common.log_config import logger +from fedn.network.combiner.hooks.safe_builtins import safe_builtins from fedn.network.combiner.modelservice import load_model_from_BytesIO, serialize_model_to_BytesIO -from fedn.network.hooks.safe_builtins import safe_builtins from fedn.utils.helpers.plugins.numpyhelper import Helper CHUNK_SIZE = 1024 * 1024 @@ -22,7 +23,7 @@ def __init__(self) -> None: "__builtins__": self.safe_builtins, } self.helper = Helper() - self.parameters = [] + self.client_results = [] def ExecuteFunction(self, request, context): # Compile the function code @@ -32,16 +33,20 @@ def ExecuteFunction(self, request, context): return fedn.FunctionResponse(result_string="Instansiated hook functions.") if request.task == "store_parameters": logger.info("Executing aggregate function.") - payload = load_model_from_BytesIO(request.payload_bytes, self.helper) - self.parameters.append(payload) + parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) + self.client_results.append((parameters, json.loads(request.payload_string))) return fedn.FunctionResponse(result_string="Stored parameters") if request.task == "aggregate": logger.info("Executing aggregate function.") - payload = load_model_from_BytesIO(request.payload_bytes, self.helper) - self.parameters.append(payload) + parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) + self.client_results.append((parameters, json.loads(request.payload_string))) result = self.execute_function_code() result_bytes = serialize_model_to_BytesIO(result, self.helper).getvalue() return fedn.FunctionResponse(result_bytes=result_bytes) + if request.task == "get_model_metadata": + model_metadata = self.get_model_metadata() + json_model_metadata = json.dumps(model_metadata, indent=4) + return fedn.FunctionResponse(result_string=json_model_metadata) def init_hook_object(self, class_code): # Prepare the globals dictionary with restricted builtins @@ -58,14 +63,23 @@ def init_hook_object(self, class_code): def execute_function_code(self): if not hasattr(self, "globals_dict"): raise AttributeError("Function provider code has not been provided.") - self.globals_dict["parameters"] = self.parameters + self.globals_dict["client_results"] = self.client_results instance_code = """ -res = function_provider.aggregate(parameters) +res = function_provider.aggregate(client_results) """ exec(instance_code, self.globals_dict) # noqa: S102 - self.parameters = [] + self.client_results = [] return self.globals_dict["res"] + def get_model_metadata(self): + if not hasattr(self, "globals_dict"): + raise AttributeError("Function provider code has not been provided.") + instance_code = """ +model_metadata = function_provider.get_model_metadata() +""" + exec(instance_code, self.globals_dict) # noqa: S102 + return self.globals_dict["model_metadata"] + def serve(): server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) diff --git a/fedn/network/hooks/safe_builtins.py b/fedn/network/combiner/hooks/safe_builtins.py similarity index 96% rename from fedn/network/hooks/safe_builtins.py rename to fedn/network/combiner/hooks/safe_builtins.py index bc8bd0aa9..8c989aaf2 100644 --- a/fedn/network/hooks/safe_builtins.py +++ b/fedn/network/combiner/hooks/safe_builtins.py @@ -57,13 +57,11 @@ "__build_class__": builtins.__build_class__, "__name__": builtins.__name__, "__import__": builtins.__import__, - # Adding common exceptions "BaseException": BaseException, "Exception": Exception, "ArithmeticError": ArithmeticError, "BufferError": BufferError, "LookupError": LookupError, - # Adding common modules "math": __import__("math"), "itertools": __import__("itertools"), "functools": __import__("functools"), diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 8f2b857bf..c950c5ef7 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -423,6 +423,7 @@ def run(self, polling_interval=1.0): if ready: if round_config["task"] == "training": tic = time.time() + round_config["model_metadata"] = self.aggregator.get_model_metadata() round_meta = self.execute_training_round(round_config) round_meta["time_exec_training"] = time.time() - tic round_meta["status"] = "Success" diff --git a/fedn/network/controller/control.py b/fedn/network/controller/control.py index 708ce4440..4386edd17 100644 --- a/fedn/network/controller/control.py +++ b/fedn/network/controller/control.py @@ -129,6 +129,8 @@ def start_session(self, session_id: str, rounds: int, round_timeout: int) -> Non for combiner in self.network.get_combiners(): combiner.set_aggregator(aggregator) + if session_config["function_provider"] is not None: + combiner.set_function_provider(session_config["function_provider"]) self.set_session_status(session_id, "Started") diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index fc5ec8b65..92ac7e259 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -247,10 +247,8 @@ service Combiner { message FunctionRequest { string task = 1; - oneof payload { - string payload_string = 2; - bytes payload_bytes = 3; - } + string payload_string = 2; + bytes payload_bytes = 3; } message FunctionResponse { diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index 36a975d4a..eca6f5965 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -15,25 +15,25 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"]\n\x0f\x46unctionRequest\x12\x0c\n\x04task\x18\x01 \x01(\t\x12\x18\n\x0epayload_string\x18\x02 \x01(\tH\x00\x12\x17\n\rpayload_bytes\x18\x03 \x01(\x0cH\x00\x42\t\n\x07payload\"M\n\x10\x46unctionResponse\x12\x17\n\rresult_string\x18\x02 \x01(\tH\x00\x12\x16\n\x0cresult_bytes\x18\x03 \x01(\x0cH\x00\x42\x08\n\x06result*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbc\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x42\n\x13SetFunctionProvider\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2S\n\x0f\x46unctionService\x12@\n\x0f\x45xecuteFunction\x12\x15.fedn.FunctionRequest\x1a\x16.fedn.FunctionResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"N\n\x0f\x46unctionRequest\x12\x0c\n\x04task\x18\x01 \x01(\t\x12\x16\n\x0epayload_string\x18\x02 \x01(\t\x12\x15\n\rpayload_bytes\x18\x03 \x01(\x0c\"M\n\x10\x46unctionResponse\x12\x17\n\rresult_string\x18\x02 \x01(\tH\x00\x12\x16\n\x0cresult_bytes\x18\x03 \x01(\x0cH\x00\x42\x08\n\x06result*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbc\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x42\n\x13SetFunctionProvider\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2S\n\x0f\x46unctionService\x12@\n\x0f\x45xecuteFunction\x12\x15.fedn.FunctionRequest\x1a\x16.fedn.FunctionResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'fedn.network.grpc.fedn_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_STATUSTYPE']._serialized_start=2506 - _globals['_STATUSTYPE']._serialized_end=2638 - _globals['_QUEUE']._serialized_start=2640 - _globals['_QUEUE']._serialized_end=2676 - _globals['_MODELSTATUS']._serialized_start=2678 - _globals['_MODELSTATUS']._serialized_end=2761 - _globals['_ROLE']._serialized_start=2763 - _globals['_ROLE']._serialized_end=2819 - _globals['_COMMAND']._serialized_start=2821 - _globals['_COMMAND']._serialized_end=2895 - _globals['_CONNECTIONSTATUS']._serialized_start=2897 - _globals['_CONNECTIONSTATUS']._serialized_end=2970 + _globals['_STATUSTYPE']._serialized_start=2491 + _globals['_STATUSTYPE']._serialized_end=2623 + _globals['_QUEUE']._serialized_start=2625 + _globals['_QUEUE']._serialized_end=2661 + _globals['_MODELSTATUS']._serialized_start=2663 + _globals['_MODELSTATUS']._serialized_end=2746 + _globals['_ROLE']._serialized_start=2748 + _globals['_ROLE']._serialized_end=2804 + _globals['_COMMAND']._serialized_start=2806 + _globals['_COMMAND']._serialized_end=2880 + _globals['_CONNECTIONSTATUS']._serialized_start=2882 + _globals['_CONNECTIONSTATUS']._serialized_end=2955 _globals['_RESPONSE']._serialized_start=71 _globals['_RESPONSE']._serialized_end=129 _globals['_STATUS']._serialized_start=132 @@ -79,19 +79,19 @@ _globals['_CONNECTIONRESPONSE']._serialized_start=2269 _globals['_CONNECTIONRESPONSE']._serialized_end=2329 _globals['_FUNCTIONREQUEST']._serialized_start=2331 - _globals['_FUNCTIONREQUEST']._serialized_end=2424 - _globals['_FUNCTIONRESPONSE']._serialized_start=2426 - _globals['_FUNCTIONRESPONSE']._serialized_end=2503 - _globals['_MODELSERVICE']._serialized_start=2972 - _globals['_MODELSERVICE']._serialized_end=3094 - _globals['_CONTROL']._serialized_start=3097 - _globals['_CONTROL']._serialized_end=3413 - _globals['_REDUCER']._serialized_start=3415 - _globals['_REDUCER']._serialized_end=3501 - _globals['_CONNECTOR']._serialized_start=3504 - _globals['_CONNECTOR']._serialized_end=3931 - _globals['_COMBINER']._serialized_start=3934 - _globals['_COMBINER']._serialized_end=4125 - _globals['_FUNCTIONSERVICE']._serialized_start=4127 - _globals['_FUNCTIONSERVICE']._serialized_end=4210 + _globals['_FUNCTIONREQUEST']._serialized_end=2409 + _globals['_FUNCTIONRESPONSE']._serialized_start=2411 + _globals['_FUNCTIONRESPONSE']._serialized_end=2488 + _globals['_MODELSERVICE']._serialized_start=2957 + _globals['_MODELSERVICE']._serialized_end=3079 + _globals['_CONTROL']._serialized_start=3082 + _globals['_CONTROL']._serialized_end=3398 + _globals['_REDUCER']._serialized_start=3400 + _globals['_REDUCER']._serialized_end=3486 + _globals['_CONNECTOR']._serialized_start=3489 + _globals['_CONNECTOR']._serialized_end=3916 + _globals['_COMBINER']._serialized_start=3919 + _globals['_COMBINER']._serialized_end=4110 + _globals['_FUNCTIONSERVICE']._serialized_start=4112 + _globals['_FUNCTIONSERVICE']._serialized_end=4195 # @@protoc_insertion_point(module_scope) diff --git a/fedn/utils/helpers/helpers.py b/fedn/utils/helpers/helpers.py index 7fbf83a81..1e5e81d75 100644 --- a/fedn/utils/helpers/helpers.py +++ b/fedn/utils/helpers/helpers.py @@ -29,6 +29,19 @@ def save_metadata(metadata, filename): json.dump(metadata, outfile) +def load_metadata(filename): + """Load metadata from file. + + :param filename: The name of the file to load from. + :type filename: str + :return: The loaded metadata. + :rtype: dict + """ + with open(filename + "-metadata", "r") as infile: + metadata = json.load(infile) + return metadata + + def save_metrics(metrics, filename): """Save metrics to file. From af36e6fecf569b14a72cf080cba825f187c8405d Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Mon, 2 Sep 2024 10:12:17 +0200 Subject: [PATCH 06/21] linter fix --- examples/custom-aggregator/client/aggregator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/custom-aggregator/client/aggregator.py b/examples/custom-aggregator/client/aggregator.py index a4816623b..40aa04ee4 100644 --- a/examples/custom-aggregator/client/aggregator.py +++ b/examples/custom-aggregator/client/aggregator.py @@ -5,9 +5,8 @@ class FunctionProvider(FunctionProviderBase): - """A FunctionProvider class responsible for aggregating model parameters - from multiple clients and performing hyperparameter tuning by adjusting - the learning rate every 20th round. The class logs the current state of + """A FunctionProvider class responsible for aggregating client model parameters and performing + hyperparameter tuning by adjusting the learning rate every 20th round. The class logs the current state of the model, learning rate, and round to facilitate monitoring and evaluation. """ From 756e8de9e2740e68f7bfc639db74b5bb7ff4d82b Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Mon, 2 Sep 2024 17:03:10 +0200 Subject: [PATCH 07/21] add proper error logging --- fedn/network/combiner/hooks/hooks.py | 46 ++++++++++++++++------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/fedn/network/combiner/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py index 55ba08a6e..98f4d9f25 100644 --- a/fedn/network/combiner/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -27,26 +27,32 @@ def __init__(self) -> None: def ExecuteFunction(self, request, context): # Compile the function code - if request.task == "setup": - logger.info("Adding function provider.") - self.init_hook_object(request.payload_string) - return fedn.FunctionResponse(result_string="Instansiated hook functions.") - if request.task == "store_parameters": - logger.info("Executing aggregate function.") - parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) - self.client_results.append((parameters, json.loads(request.payload_string))) - return fedn.FunctionResponse(result_string="Stored parameters") - if request.task == "aggregate": - logger.info("Executing aggregate function.") - parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) - self.client_results.append((parameters, json.loads(request.payload_string))) - result = self.execute_function_code() - result_bytes = serialize_model_to_BytesIO(result, self.helper).getvalue() - return fedn.FunctionResponse(result_bytes=result_bytes) - if request.task == "get_model_metadata": - model_metadata = self.get_model_metadata() - json_model_metadata = json.dumps(model_metadata, indent=4) - return fedn.FunctionResponse(result_string=json_model_metadata) + try: + if request.task == "setup": + logger.info("Adding function provider.") + self.init_hook_object(request.payload_string) + return fedn.FunctionResponse(result_string="Instansiated hook functions.") + if request.task == "store_parameters": + logger.info("Executing aggregate function.") + parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) + self.client_results.append((parameters, json.loads(request.payload_string))) + return fedn.FunctionResponse(result_string="Stored parameters") + if request.task == "aggregate": + logger.info("Executing aggregate function.") + parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) + self.client_results.append((parameters, json.loads(request.payload_string))) + result = self.execute_function_code() + result_bytes = serialize_model_to_BytesIO(result, self.helper).getvalue() + return fedn.FunctionResponse(result_bytes=result_bytes) + if request.task == "get_model_metadata": + model_metadata = self.get_model_metadata() + json_model_metadata = json.dumps(model_metadata, indent=4) + return fedn.FunctionResponse(result_string=json_model_metadata) + except Exception as e: + logger.error(f"Error executing task {request.task}: {e}", exc_info=True) + context.set_details(f"Error: {e}") + context.set_code(grpc.StatusCode.INTERNAL) + return fedn.FunctionResponse(result_string=f"Error executing task: {e}") def init_hook_object(self, class_code): # Prepare the globals dictionary with restricted builtins From dd9646cdbb3af22f85b792dbe5f596034668ddf4 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 09:06:54 +0200 Subject: [PATCH 08/21] Refactor hooks and updatehandler --- examples/custom-aggregator/README.rst | 8 - .../custom-aggregator/client/aggregator.py | 112 ---------- .../.dockerignore | 0 .../.gitignore | 0 examples/server-functions/README.rst | 11 + .../client/data.py | 0 .../client/fedn.yaml | 0 .../client/model.py | 0 .../client/predict.py | 0 .../client/python_env.yaml | 0 .../client/train.py | 16 +- .../client/validate.py | 0 .../docker-compose.override.yaml | 0 examples/server-functions/server_functions.py | 47 ++++ fedn/network/api/client.py | 15 +- fedn/network/api/interface.py | 10 +- fedn/network/clients/client.py | 9 +- .../combiner/aggregators/aggregatorbase.py | 120 +---------- fedn/network/combiner/aggregators/custom.py | 98 --------- fedn/network/combiner/aggregators/fedavg.py | 30 +-- fedn/network/combiner/aggregators/fedopt.py | 73 +++---- .../aggregators/functionproviderbase.py | 63 ------ .../combiner/aggregators/tests/test_fedavg.py | 4 +- fedn/network/combiner/combiner.py | 10 +- fedn/network/combiner/hook_client.py | 49 ----- fedn/network/combiner/hooks/allowed_import.py | 7 + fedn/network/combiner/hooks/hook_client.py | 76 +++++++ fedn/network/combiner/hooks/hooks.py | 181 ++++++++++------ fedn/network/combiner/hooks/safe_builtins.py | 72 ------- .../combiner/hooks/serverfunctionsbase.py | 74 +++++++ fedn/network/combiner/interfaces.py | 8 +- fedn/network/combiner/modelservice.py | 35 ++- fedn/network/combiner/roundhandler.py | 126 +++-------- fedn/network/combiner/updatehandler.py | 204 ++++++++++++++++++ fedn/network/controller/control.py | 14 +- fedn/network/grpc/fedn.proto | 58 +++-- fedn/network/grpc/fedn_pb2.py | 78 ++++--- fedn/network/grpc/fedn_pb2_grpc.py | 172 +++++++++++++-- 38 files changed, 927 insertions(+), 853 deletions(-) delete mode 100644 examples/custom-aggregator/README.rst delete mode 100644 examples/custom-aggregator/client/aggregator.py rename examples/{custom-aggregator => server-functions}/.dockerignore (100%) rename examples/{custom-aggregator => server-functions}/.gitignore (100%) create mode 100644 examples/server-functions/README.rst rename examples/{custom-aggregator => server-functions}/client/data.py (100%) rename examples/{custom-aggregator => server-functions}/client/fedn.yaml (100%) rename examples/{custom-aggregator => server-functions}/client/model.py (100%) rename examples/{custom-aggregator => server-functions}/client/predict.py (100%) rename examples/{custom-aggregator => server-functions}/client/python_env.yaml (100%) rename examples/{custom-aggregator => server-functions}/client/train.py (87%) rename examples/{custom-aggregator => server-functions}/client/validate.py (100%) rename examples/{custom-aggregator => server-functions}/docker-compose.override.yaml (100%) create mode 100644 examples/server-functions/server_functions.py delete mode 100644 fedn/network/combiner/aggregators/custom.py delete mode 100644 fedn/network/combiner/aggregators/functionproviderbase.py delete mode 100644 fedn/network/combiner/hook_client.py create mode 100644 fedn/network/combiner/hooks/allowed_import.py create mode 100644 fedn/network/combiner/hooks/hook_client.py delete mode 100644 fedn/network/combiner/hooks/safe_builtins.py create mode 100644 fedn/network/combiner/hooks/serverfunctionsbase.py create mode 100644 fedn/network/combiner/updatehandler.py diff --git a/examples/custom-aggregator/README.rst b/examples/custom-aggregator/README.rst deleted file mode 100644 index 33ffbaa06..000000000 --- a/examples/custom-aggregator/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -FEDn Project: Custom aggregator (hyperparameter tuning) ------------------------------ - -Will be updated after studio update. - -To run custom aggregators: - -client.start_session(aggregator="custom", function_provider_path="client/aggregator.py", rounds=101) diff --git a/examples/custom-aggregator/client/aggregator.py b/examples/custom-aggregator/client/aggregator.py deleted file mode 100644 index 40aa04ee4..000000000 --- a/examples/custom-aggregator/client/aggregator.py +++ /dev/null @@ -1,112 +0,0 @@ -import numpy as np - -from fedn.common.log_config import logger -from fedn.network.combiner.aggregators.functionproviderbase import FunctionProviderBase - - -class FunctionProvider(FunctionProviderBase): - """A FunctionProvider class responsible for aggregating client model parameters and performing - hyperparameter tuning by adjusting the learning rate every 20th round. The class logs the current state of - the model, learning rate, and round to facilitate monitoring and evaluation. - """ - - def __init__(self) -> None: - self.current_round = -1 - self.initial_parameters = None - self.learning_rates = [0.001, 0.01, 0.0001, 0.1, 0.00001] - self.current_lr_index = -1 - self.current_lr = 0 # start with 0 learning rate the first round to get initial parameters - self.current_parameters = None - - # Tracking metrics - self.highest_accuracy = 0 - self.highest_accuracy_round = -1 - self.highest_accuracy_lr = 0 - self.mean_loss_per_lr = [] - self.mean_acc_per_lr = [] - self.highest_mean_acc = 0 - self.highest_mean_acc_round = -1 - self.highest_mean_acc_lr = None - - def aggregate(self, results: list[tuple[list[np.ndarray], dict]]) -> list[np.ndarray]: - """Aggregate model parameters using weighted average based on the number of examples each client has. - - Args: - ---- - results (list of tuples): Each tuple contains: - - A list of numpy.ndarrays representing model parameters from a client. - - A dictionary containing client metadata, which must include a key "num_examples" indicating - the number of examples used by the client. - - Returns: - ------- - list of numpy.ndarrays: Aggregated model parameters as a list of numpy.ndarrays. - - """ - total_loss = 0 - total_acc = 0 - num_clients = len(results) - if self.current_round == -1: - self.initial_parameters = results[0][0] - averaged_parameters = self.initial_parameters # first round no updates were made. - elif self.current_round % 20 == 0: - if self.mean_loss_per_lr: - logger.info(f"Completed Learning Rate: {self.current_lr}") - logger.info(f"Mean Loss: {np.mean(self.mean_loss_per_lr)}, Highest Accuracy: {np.max(self.mean_acc_per_lr)}") - logger.info( - f"""Highest mean accuracy across rounds: {self.highest_mean_acc} - at round {self.highest_mean_acc_round} with lr {self.highest_mean_acc_lr}""" - ) - - # Reset tracking for the new learning rate - self.mean_loss_per_lr = [] - self.mean_acc_per_lr = [] - - averaged_parameters = self.initial_parameters - self.current_lr_index += 1 - self.current_lr = self.learning_rates[self.current_lr_index] - else: - # Aggregate using fedavg - summed_parameters = [np.zeros_like(param) for param in results[0][0]] - total_weight = 0 - for client_params, client_metadata in results: - weight = client_metadata.get("num_examples", 1) - total_weight += weight - for i, param in enumerate(client_params): - summed_parameters[i] += param * weight - - total_loss += client_metadata.get("test_loss", 0) - total_acc += client_metadata.get("test_acc", 0) - - averaged_parameters = [param / total_weight for param in summed_parameters] - - # Calculate average loss and accuracy by number of clients - avg_loss = total_loss / num_clients if num_clients > 0 else 0 - avg_acc = total_acc / num_clients if num_clients > 0 else 0 - - # Update the tracking for the current learning rate - self.mean_loss_per_lr.append(avg_loss) - self.mean_acc_per_lr.append(avg_acc) - - # Check if we have a new highest accuracy - if avg_acc > self.highest_accuracy: - self.highest_accuracy = avg_acc - self.highest_accuracy_round = self.current_round - self.highest_accuracy_lr = self.current_lr - - # Check if we have a new highest mean accuracy across rounds - if avg_acc > self.highest_mean_acc: - self.highest_mean_acc = avg_acc - self.highest_mean_acc_round = self.current_round - self.highest_mean_acc_lr = self.current_lr - - # Print the metrics - logger.info(f"Round {self.current_round} - Learning Rate: {self.current_lr}") - logger.info(f"Average Test Loss: {avg_loss}, Average Test Accuracy: {avg_acc}") - logger.info(f"Highest Accuracy Achieved: {self.highest_accuracy} at round {self.highest_accuracy_round} with lr {self.highest_accuracy_lr}") - - self.current_round += 1 - return averaged_parameters - - def get_model_metadata(self): - return {"learning_rate": self.current_lr, "parameter_tuning": True} diff --git a/examples/custom-aggregator/.dockerignore b/examples/server-functions/.dockerignore similarity index 100% rename from examples/custom-aggregator/.dockerignore rename to examples/server-functions/.dockerignore diff --git a/examples/custom-aggregator/.gitignore b/examples/server-functions/.gitignore similarity index 100% rename from examples/custom-aggregator/.gitignore rename to examples/server-functions/.gitignore diff --git a/examples/server-functions/README.rst b/examples/server-functions/README.rst new file mode 100644 index 000000000..43c9c2fb4 --- /dev/null +++ b/examples/server-functions/README.rst @@ -0,0 +1,11 @@ +FEDn Project: Server functions toy example +----------------------------- + +See server_functions.py for details. + +README Will be updated after studio update. + +To run with server functions: + +from server_functions import ServerFunctions +client.start_session(aggregator="custom", server_functions=ServerFunctions) diff --git a/examples/custom-aggregator/client/data.py b/examples/server-functions/client/data.py similarity index 100% rename from examples/custom-aggregator/client/data.py rename to examples/server-functions/client/data.py diff --git a/examples/custom-aggregator/client/fedn.yaml b/examples/server-functions/client/fedn.yaml similarity index 100% rename from examples/custom-aggregator/client/fedn.yaml rename to examples/server-functions/client/fedn.yaml diff --git a/examples/custom-aggregator/client/model.py b/examples/server-functions/client/model.py similarity index 100% rename from examples/custom-aggregator/client/model.py rename to examples/server-functions/client/model.py diff --git a/examples/custom-aggregator/client/predict.py b/examples/server-functions/client/predict.py similarity index 100% rename from examples/custom-aggregator/client/predict.py rename to examples/server-functions/client/predict.py diff --git a/examples/custom-aggregator/client/python_env.yaml b/examples/server-functions/client/python_env.yaml similarity index 100% rename from examples/custom-aggregator/client/python_env.yaml rename to examples/server-functions/client/python_env.yaml diff --git a/examples/custom-aggregator/client/train.py b/examples/server-functions/client/train.py similarity index 87% rename from examples/custom-aggregator/client/train.py rename to examples/server-functions/client/train.py index 18b61008b..7d87e680e 100644 --- a/examples/custom-aggregator/client/train.py +++ b/examples/server-functions/client/train.py @@ -11,8 +11,8 @@ # swap this to the load_metadata from helpers.helpers on release.. -def load_metadata(filename): - """Load metadata from file. +def load_client_config(filename): + """Load client_config from file. :param filename: The name of the file to load from. :type filename: str @@ -68,14 +68,8 @@ def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1 # Load parmeters and initialize model model = load_parameters(in_model_path) - model_metadata = load_metadata(in_model_path) - lr = model_metadata["learning_rate"] - - # validate each new aggregated model for hyperparameter tuning - if model_metadata["parameter_tuning"]: - test_loss, test_acc = validate(model, data_path) - else: - test_loss, test_acc = torch.tensor(0.0), torch.tensor(0.0) + client_config = load_client_config(in_model_path) + lr = client_config["learning_rate"] # Train optimizer = torch.optim.SGD(model.parameters(), lr=lr) @@ -103,8 +97,6 @@ def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1 "batch_size": batch_size, "epochs": epochs, "lr": lr, - "test_loss": test_loss.item(), - "test_acc": test_acc.item(), } # Save JSON metadata file (mandatory) diff --git a/examples/custom-aggregator/client/validate.py b/examples/server-functions/client/validate.py similarity index 100% rename from examples/custom-aggregator/client/validate.py rename to examples/server-functions/client/validate.py diff --git a/examples/custom-aggregator/docker-compose.override.yaml b/examples/server-functions/docker-compose.override.yaml similarity index 100% rename from examples/custom-aggregator/docker-compose.override.yaml rename to examples/server-functions/docker-compose.override.yaml diff --git a/examples/server-functions/server_functions.py b/examples/server-functions/server_functions.py new file mode 100644 index 000000000..65a75db83 --- /dev/null +++ b/examples/server-functions/server_functions.py @@ -0,0 +1,47 @@ +from fedn.common.log_config import logger +from fedn.network.combiner.hooks.allowed_import import Dict, List, ServerFunctionsBase, Tuple, np, random + +# See allowed_imports for what packages you can use in this class. + + +class ServerFunctions(ServerFunctionsBase): + # toy example to highlight functionality of ServerFunctions. + def __init__(self) -> None: + # You can keep a state between different functions to have them work together. + self.round = 0 + self.lr = 0.01 + + # Skip any function to use the default FEDn implementation for the function. + + # Called first in the beggining of a round to select clients. + def client_selection(self, client_ids: list[str]) -> list: + # Pick 10 random clients + client_ids = random.sample(client_ids, min(len(client_ids), 10)) # noqa: F405 + return client_ids + + # Called secondly before sending the global model. + def client_config(self, global_model: list[np.ndarray]) -> dict: + # Decrease learning rate every 10 rounds + if self.round % 10 == 0: + self.lr = self.lr * 0.1 + # see client/train.py for how to load the client config. + self.round += 1 + return {"learning_rate": self.lr} + + # Called third to aggregate the client updates. + def aggregate(self, previous_global: List[np.ndarray], client_updates: Dict[str, Tuple[List[np.ndarray], dict]]) -> List[np.ndarray]: + # Weighted fedavg implementation. + weighted_sum = [np.zeros_like(param) for param in previous_global] + total_weight = 0 + for client_id, (client_parameters, metadata) in client_updates.items(): + num_examples = metadata.get("num_examples", 0) + if num_examples > 0: + total_weight += num_examples + for i in range(len(weighted_sum)): + weighted_sum[i] += client_parameters[i] * num_examples + if total_weight > 0: + averaged_updates = [weighted / total_weight for weighted in weighted_sum] + return averaged_updates + # Need to use logger instead of prints. + logger.info("Aggregation failed, using previous global model.") + return previous_global diff --git a/fedn/network/api/client.py b/fedn/network/api/client.py index 8d2024e21..ab3e2e07d 100644 --- a/fedn/network/api/client.py +++ b/fedn/network/api/client.py @@ -1,7 +1,10 @@ +import inspect import os import requests +from fedn.network.combiner.hooks.serverfunctionsbase import ServerFunctionsBase + __all__ = ["APIClient"] @@ -574,7 +577,7 @@ def start_session( helper: str = "", min_clients: int = 1, requested_clients: int = 8, - function_provider_path: str = None, + server_functions: ServerFunctionsBase = None, ): """Start a new session. @@ -618,7 +621,7 @@ def start_session( "helper": helper, "min_clients": min_clients, "requested_clients": requested_clients, - "function_provider": function_provider_path if function_provider_path is None else self._read_function_provider(function_provider_path), + "server_functions": None if server_functions is None else inspect.getsource(server_functions), }, verify=self.verify, headers=self.headers, @@ -789,11 +792,3 @@ def get_validations_count(self): _json = response.json() return _json - - def _read_function_provider(self, path): - # Open the file in read mode - with open(path, "r") as file: - file_contents = file.read() - - # Print the file contents - return file_contents diff --git a/fedn/network/api/interface.py b/fedn/network/api/interface.py index 5fa771b52..8a83d7464 100644 --- a/fedn/network/api/interface.py +++ b/fedn/network/api/interface.py @@ -11,7 +11,7 @@ from fedn.common.config import get_controller_config, get_network_config from fedn.common.log_config import logger from fedn.network.combiner.interfaces import CombinerInterface, CombinerUnavailableError -from fedn.network.combiner.modelservice import load_model_from_BytesIO +from fedn.network.combiner.modelservice import load_model_from_bytes from fedn.network.state import ReducerState, ReducerStateToString from fedn.utils.checksum import sha @@ -656,9 +656,9 @@ def set_initial_model(self, file): file_bytes.write(file.read()) file_bytes.seek(0) - # Load the model using the load_model_from_BytesIO function + # Load the model using the load_model_from_bytes function model_bytes = file_bytes.read() - model = load_model_from_BytesIO(model_bytes, helper) + model = load_model_from_bytes(model_bytes, helper) self.control.commit(file.filename, model) except Exception as e: logger.debug(e) @@ -945,7 +945,7 @@ def start_session( helper="", min_clients=1, requested_clients=8, - function_provider=None, + server_functions=None, ): """Start a session. @@ -1048,7 +1048,7 @@ def start_session( "task": (""), "validate": validate, "helper_type": helper, - "function_provider": function_provider, + "server_functions": server_functions, } # Start session diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index 2cb37bcaf..a73754a09 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -484,7 +484,7 @@ def _listen_to_task_stream(self): if not self._connected: return - def _process_training_request(self, model_id: str, session_id: str = None, model_metadata: dict = None): + def _process_training_request(self, model_id: str, session_id: str = None, client_config: dict = None): """Process a training (model update) request. :param model_id: The model id of the model to be updated. @@ -510,7 +510,7 @@ def _process_training_request(self, model_id: str, session_id: str = None, model with open(inpath, "wb") as fh: fh.write(mdl.getbuffer()) - save_metadata(metadata=model_metadata, filename=inpath) + save_metadata(metadata=client_config, filename=inpath) outpath = self.helper.get_tmp_path() tic = time.time() @@ -651,8 +651,8 @@ def process_request(self): if task_type == "train": tic = time.time() self.state = ClientState.training - model_metadata = json.loads(request.data)["model_metadata"] - model_id, meta = self._process_training_request(request.model_id, session_id=request.session_id, model_metadata=model_metadata) + client_config = json.loads(request.data)["client_config"] + model_id, meta = self._process_training_request(request.model_id, session_id=request.session_id, client_config=client_config) if meta is not None: processing_time = time.time() - tic @@ -663,6 +663,7 @@ def process_request(self): # Send model update to combiner update = fedn.ModelUpdate() update.sender.name = self.name + update.sender.client_id = self.id update.sender.role = fedn.WORKER update.receiver.name = request.sender.name update.receiver.role = request.sender.role diff --git a/fedn/network/combiner/aggregators/aggregatorbase.py b/fedn/network/combiner/aggregators/aggregatorbase.py index 4014106a1..19006d8a8 100644 --- a/fedn/network/combiner/aggregators/aggregatorbase.py +++ b/fedn/network/combiner/aggregators/aggregatorbase.py @@ -1,10 +1,8 @@ import importlib -import json import queue -import traceback from abc import ABC, abstractmethod -from fedn.common.log_config import logger +from fedn.network.combiner.updatehandler import UpdateHandler AGGREGATOR_PLUGIN_PATH = "fedn.network.combiner.aggregators.{}" @@ -12,27 +10,15 @@ class AggregatorBase(ABC): """Abstract class defining an aggregator. - :param id: A reference to id of :class: `fedn.network.combiner.Combiner` - :type id: str - :param storage: Model repository for :class: `fedn.network.combiner.Combiner` - :type storage: class: `fedn.common.storage.s3.s3repo.S3ModelRepository` - :param server: A handle to the Combiner class :class: `fedn.network.combiner.Combiner` - :type server: class: `fedn.network.combiner.Combiner` - :param modelservice: A handle to the model service :class: `fedn.network.combiner.modelservice.ModelService` - :type modelservice: class: `fedn.network.combiner.modelservice.ModelService` - :param control: A handle to the :class: `fedn.network.combiner.roundhandler.RoundHandler` - :type control: class: `fedn.network.combiner.roundhandler.RoundHandler` + :param control: A handle to the :class: `fedn.network.combiner.updatehandler.UpdateHandler` + :type control: class: `fedn.network.combiner.updatehandler.UpdateHandler` """ @abstractmethod - def __init__(self, storage, server, modelservice, round_handler): + def __init__(self, update_handler: UpdateHandler): """Initialize the aggregator.""" self.name = self.__class__.__name__ - self.storage = storage - self.server = server - self.modelservice = modelservice - self.round_handler = round_handler - self.model_updates = queue.Queue() + self.update_handler = update_handler @abstractmethod def combine_models(self, nr_expected_models=None, nr_required_models=1, helper=None, timeout=180, delete_models=True, parameters=None): @@ -55,100 +41,8 @@ def combine_models(self, nr_expected_models=None, nr_required_models=1, helper=N """ pass - def get_model_metadata(self): - """Sends metadata to the clients.""" - return {} - - def on_model_update(self, model_update): - """Callback when a new client model update is recieved. - - Performs (optional) validation and pre-processing, - and then puts the update id on the aggregation queue. - Override in subclass as needed. - - :param model_update: fedn.network.grpc.fedn.proto.ModelUpdate - :type model_id: str - """ - try: - logger.info("AGGREGATOR({}): callback received model update {}".format(self.name, model_update.model_update_id)) - - # Validate the update and metadata - valid_update = self._validate_model_update(model_update) - if valid_update: - # Push the model update to the processing queue - self.model_updates.put(model_update) - else: - logger.warning("AGGREGATOR({}): Invalid model update, skipping.".format(self.name)) - except Exception as e: - tb = traceback.format_exc() - logger.error("AGGREGATOR({}): failed to receive model update: {}".format(self.name, e)) - logger.error(tb) - pass - - def _validate_model_update(self, model_update): - """Validate the model update. - - :param model_update: A ModelUpdate message. - :type model_update: object - :return: True if the model update is valid, False otherwise. - :rtype: bool - """ - try: - data = json.loads(model_update.meta)["training_metadata"] - _ = data["num_examples"] - except KeyError: - tb = traceback.format_exc() - logger.error("AGGREGATOR({}): Invalid model update, missing metadata.".format(self.name)) - logger.error(tb) - return False - return True - - def next_model_update(self): - """Get the next model update from the queue. - - :param helper: A helper object. - :type helper: object - :return: The model update. - :rtype: fedn.network.grpc.fedn.proto.ModelUpdate - """ - model_update = self.model_updates.get(block=False) - return model_update - - def load_model_update(self, model_update, helper): - """Load the memory representation of the model update. - - Load the model update paramters and the - associate metadata into memory. - - :param model_update: The model update. - :type model_update: fedn.network.grpc.fedn.proto.ModelUpdate - :param helper: A helper object. - :type helper: fedn.utils.helpers.helperbase.Helper - :return: A tuple of (parameters, metadata) - :rtype: tuple - """ - model_id = model_update.model_update_id - model = self.round_handler.load_model_update(helper, model_id) - # Get relevant metadata - metadata = json.loads(model_update.meta) - if "config" in metadata.keys(): - # Used in Python client - config = json.loads(metadata["config"]) - else: - # Used in C++ client - config = json.loads(model_update.config) - training_metadata = metadata["training_metadata"] - training_metadata["round_id"] = config["round_id"] - - return model, training_metadata - - def get_state(self): - """Get the state of the aggregator's queue, including the number of model updates.""" - state = {"queue_len": self.model_updates.qsize()} - return state - -def get_aggregator(aggregator_module_name, storage, server, modelservice, control): +def get_aggregator(aggregator_module_name, update_handler): """Return an instance of the helper class. :param helper_module_name: The name of the helper plugin module. @@ -166,4 +60,4 @@ def get_aggregator(aggregator_module_name, storage, server, modelservice, contro """ aggregator_plugin = AGGREGATOR_PLUGIN_PATH.format(aggregator_module_name) aggregator = importlib.import_module(aggregator_plugin) - return aggregator.Aggregator(storage, server, modelservice, control) + return aggregator.Aggregator(update_handler) diff --git a/fedn/network/combiner/aggregators/custom.py b/fedn/network/combiner/aggregators/custom.py deleted file mode 100644 index c56dc4b73..000000000 --- a/fedn/network/combiner/aggregators/custom.py +++ /dev/null @@ -1,98 +0,0 @@ -import traceback - -from fedn.common.log_config import logger -from fedn.network.combiner.aggregators.aggregatorbase import AggregatorBase - - -class Aggregator(AggregatorBase): - """Custom aggregator provided from user defined code. - - :param id: A reference to id of :class: `fedn.network.combiner.Combiner` - :type id: str - :param storage: Model repository for :class: `fedn.network.combiner.Combiner` - :type storage: class: `fedn.common.storage.s3.s3repo.S3ModelRepository` - :param server: A handle to the Combiner class :class: `fedn.network.combiner.Combiner` - :type server: class: `fedn.network.combiner.Combiner` - :param modelservice: A handle to the model service :class: `fedn.network.combiner.modelservice.ModelService` - :type modelservice: class: `fedn.network.combiner.modelservice.ModelService` - :param control: A handle to the :class: `fedn.network.combiner.roundhandler.RoundHandler` - :type control: class: `fedn.network.combiner.roundhandler.RoundHandler` - - """ - - def __init__(self, storage, server, modelservice, round_handler): - """Constructor method""" - super().__init__(storage, server, modelservice, round_handler) - - self.name = "custom" - self.code_set = False - - def get_model_metadata(self): - """Sends metadata to the clients.""" - if not self.code_set: - self.round_handler.set_function_provider() - self.code_set = True - return self.round_handler.combiner_hook_client.get_model_metadata() - - def combine_models(self, helper=None, delete_models=True, parameters=None): - """Aggregate all model updates with custom aggregator. - - :param helper: An instance of :class: `fedn.utils.helpers.helpers.HelperBase`, ML framework specific helper, defaults to None - :type helper: class: `fedn.utils.helpers.helpers.HelperBase`, optional - :param time_window: The time window for model aggregation, defaults to 180 - :type time_window: int, optional - :param max_nr_models: The maximum number of updates aggregated, defaults to 100 - :type max_nr_models: int, optional - :param delete_models: Delete models from storage after aggregation, defaults to True - :type delete_models: bool, optional - :return: The global model and metadata - :rtype: tuple - """ - data = {} - data["time_model_load"] = 0.0 - data["time_model_aggregation"] = 0.0 - - model = None - nr_aggregated_models = 0 - total_examples = 0 - - logger.info("AGGREGATOR({}): Aggregating model updates... ".format(self.name)) - if not self.code_set: - self.round_handler.set_function_provider() - self.code_set = True - while not self.model_updates.empty(): - try: - # Get next model from queue - logger.info("AGGREGATOR({}): Getting next model update from queue.".format(self.name)) - model_update = self.next_model_update() - - # Load model parameters and metadata - logger.info("AGGREGATOR({}): Loading model metadata {}.".format(self.name, model_update.model_update_id)) - model_next, metadata = self.load_model_update(model_update, helper) - - logger.info("AGGREGATOR({}): Processing model update {}, metadata: {} ".format(self.name, model_update.model_update_id, metadata)) - - # Increment total number of examples - total_examples += metadata["num_examples"] - - nr_aggregated_models += 1 - # Delete model from storage - if delete_models: - self.modelservice.temp_model_storage.delete(model_update.model_update_id) - logger.info("AGGREGATOR({}): Deleted model update {} from storage.".format(self.name, model_update.model_update_id)) - - self.model_updates.task_done() - if not self.model_updates.empty(): - self.round_handler.combiner_hook_client.call_function("store_parameters", model_next, helper, metadata) - else: - model = self.round_handler.combiner_hook_client.call_function("aggregate", model_next, helper, metadata) - except Exception as e: - tb = traceback.format_exc() - logger.error(f"AGGREGATOR({self.name}): Error encoutered while processing model update: {e}") - logger.error(tb) - self.model_updates.task_done() - - data["nr_aggregated_models"] = nr_aggregated_models - - logger.info("AGGREGATOR({}): Aggregation completed, aggregated {} models.".format(self.name, nr_aggregated_models)) - return model, data diff --git a/fedn/network/combiner/aggregators/fedavg.py b/fedn/network/combiner/aggregators/fedavg.py index 0d965cfa0..2ab155e67 100644 --- a/fedn/network/combiner/aggregators/fedavg.py +++ b/fedn/network/combiner/aggregators/fedavg.py @@ -8,22 +8,13 @@ class Aggregator(AggregatorBase): """Local SGD / Federated Averaging (FedAvg) aggregator. Computes a weighted mean of parameter updates. - :param id: A reference to id of :class: `fedn.network.combiner.Combiner` - :type id: str - :param storage: Model repository for :class: `fedn.network.combiner.Combiner` - :type storage: class: `fedn.common.storage.s3.s3repo.S3ModelRepository` - :param server: A handle to the Combiner class :class: `fedn.network.combiner.Combiner` - :type server: class: `fedn.network.combiner.Combiner` - :param modelservice: A handle to the model service :class: `fedn.network.combiner.modelservice.ModelService` - :type modelservice: class: `fedn.network.combiner.modelservice.ModelService` - :param control: A handle to the :class: `fedn.network.combiner.roundhandler.RoundHandler` - :type control: class: `fedn.network.combiner.roundhandler.RoundHandler` - + :param control: A handle to the :class: `fedn.network.combiner.updatehandler.UpdateHandler` + :type control: class: `fedn.network.combiner.updatehandler.UpdateHandler` """ - def __init__(self, storage, server, modelservice, round_handler): + def __init__(self, update_handler): """Constructor method""" - super().__init__(storage, server, modelservice, round_handler) + super().__init__(update_handler) self.name = "fedavg" @@ -52,15 +43,15 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): logger.info("AGGREGATOR({}): Aggregating model updates... ".format(self.name)) - while not self.model_updates.empty(): + model_updates = self.update_handler.get_model_updates() + + for model_update in model_updates: try: - # Get next model from queue logger.info("AGGREGATOR({}): Getting next model update from queue.".format(self.name)) - model_update = self.next_model_update() # Load model parameters and metadata logger.info("AGGREGATOR({}): Loading model metadata {}.".format(self.name, model_update.model_update_id)) - model_next, metadata = self.load_model_update(model_update, helper) + model_next, metadata = self.update_handler.load_model_update(model_update, helper) logger.info("AGGREGATOR({}): Processing model update {}, metadata: {} ".format(self.name, model_update.model_update_id, metadata)) @@ -75,14 +66,11 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): nr_aggregated_models += 1 # Delete model from storage if delete_models: - self.modelservice.temp_model_storage.delete(model_update.model_update_id) - logger.info("AGGREGATOR({}): Deleted model update {} from storage.".format(self.name, model_update.model_update_id)) - self.model_updates.task_done() + self.update_handler.delete_model(model_update) except Exception as e: tb = traceback.format_exc() logger.error(f"AGGREGATOR({self.name}): Error encoutered while processing model update: {e}") logger.error(tb) - self.model_updates.task_done() data["nr_aggregated_models"] = nr_aggregated_models diff --git a/fedn/network/combiner/aggregators/fedopt.py b/fedn/network/combiner/aggregators/fedopt.py index 5041e097f..9e57ed9ca 100644 --- a/fedn/network/combiner/aggregators/fedopt.py +++ b/fedn/network/combiner/aggregators/fedopt.py @@ -1,4 +1,5 @@ import math +from copy import deepcopy from fedn.common.exceptions import InvalidParameterError from fedn.common.log_config import logger @@ -6,7 +7,7 @@ class Aggregator(AggregatorBase): - """ Federated Optimization (FedOpt) aggregator. + """Federated Optimization (FedOpt) aggregator. Implmentation following: https://arxiv.org/pdf/2003.00295.pdf @@ -16,23 +17,13 @@ class Aggregator(AggregatorBase): are "adam", "yogi", "adagrad". - - :param id: A reference to id of :class: `fedn.network.combiner.Combiner` - :type id: str - :param storage: Model repository for :class: `fedn.network.combiner.Combiner` - :type storage: class: `fedn.common.storage.s3.s3repo.S3ModelRepository` - :param server: A handle to the Combiner class :class: `fedn.network.combiner.Combiner` - :type server: class: `fedn.network.combiner.Combiner` - :param modelservice: A handle to the model service :class: `fedn.network.combiner.modelservice.ModelService` - :type modelservice: class: `fedn.network.combiner.modelservice.ModelService` - :param control: A handle to the :class: `fedn.network.combiner.roundhandler.RoundHandler` - :type control: class: `fedn.network.combiner.roundhandler.RoundHandler` + :param control: A handle to the :class: `fedn.network.combiner.updatehandler.UpdateHandler` + :type control: class: `fedn.network.combiner.updatehandler.UpdateHandler` """ - def __init__(self, storage, server, modelservice, round_handler): - - super().__init__(storage, server, modelservice, round_handler) + def __init__(self, update_handler): + super().__init__(update_handler) self.name = "fedopt" # To store momentum @@ -103,42 +94,34 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): nr_aggregated_models = 0 total_examples = 0 - logger.info( - "AGGREGATOR({}): Aggregating model updates... ".format(self.name)) + logger.info("AGGREGATOR({}): Aggregating model updates... ".format(self.name)) - while not self.model_updates.empty(): - try: - # Get next model from queue - model_update = self.next_model_update() + model_updates = self.update_handler.get_model_updates() + for model_update in model_updates: + try: # Load model paratmeters and metadata - model_next, metadata = self.load_model_update(model_update, helper) + model_next, metadata = self.update_handler.load_model_update(model_update, helper) - logger.info( - "AGGREGATOR({}): Processing model update {}".format(self.name, model_update.model_update_id)) + logger.info("AGGREGATOR({}): Processing model update {}".format(self.name, model_update.model_update_id)) # Increment total number of examples total_examples += metadata["num_examples"] if nr_aggregated_models == 0: - model_old = self.round_handler.load_model_update(helper, model_update.model_id) + model_old = deepcopy(model_next) pseudo_gradient = helper.subtract(model_next, model_old) else: pseudo_gradient_next = helper.subtract(model_next, model_old) - pseudo_gradient = helper.increment_average( - pseudo_gradient, pseudo_gradient_next, metadata["num_examples"], total_examples) + pseudo_gradient = helper.increment_average(pseudo_gradient, pseudo_gradient_next, metadata["num_examples"], total_examples) nr_aggregated_models += 1 # Delete model from storage if delete_models: - self.modelservice.temp_model_storage.delete(model_update.model_update_id) - logger.info( - "AGGREGATOR({}): Deleted model update {} from storage.".format(self.name, model_update.model_update_id)) - self.model_updates.task_done() + self.update_handler.delete_model(model_update.model_update_id) + logger.info("AGGREGATOR({}): Deleted model update {} from storage.".format(self.name, model_update.model_update_id)) except Exception as e: - logger.error( - "AGGREGATOR({}): Error encoutered while processing model update {}, skipping this update.".format(self.name, e)) - self.model_updates.task_done() + logger.error("AGGREGATOR({}): Error encoutered while processing model update {}, skipping this update.".format(self.name, e)) if parameters["serveropt"] == "adam": model = self.serveropt_adam(helper, pseudo_gradient, model_old, parameters) @@ -156,7 +139,7 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): return model, data def serveropt_adam(self, helper, pseudo_gradient, model_old, parameters): - """ Server side optimization, FedAdam. + """Server side optimization, FedAdam. :param helper: instance of helper class. :type helper: Helper @@ -178,12 +161,12 @@ def serveropt_adam(self, helper, pseudo_gradient, model_old, parameters): self.v = helper.ones(pseudo_gradient, math.pow(tau, 2)) if not self.m: - self.m = helper.multiply(pseudo_gradient, [(1.0-beta1)]*len(pseudo_gradient)) + self.m = helper.multiply(pseudo_gradient, [(1.0 - beta1)] * len(pseudo_gradient)) else: - self.m = helper.add(self.m, pseudo_gradient, beta1, (1.0-beta1)) + self.m = helper.add(self.m, pseudo_gradient, beta1, (1.0 - beta1)) p = helper.power(pseudo_gradient, 2) - self.v = helper.add(self.v, p, beta2, (1.0-beta2)) + self.v = helper.add(self.v, p, beta2, (1.0 - beta2)) sv = helper.add(helper.sqrt(self.v), helper.ones(self.v, tau)) t = helper.divide(self.m, sv) @@ -192,7 +175,7 @@ def serveropt_adam(self, helper, pseudo_gradient, model_old, parameters): return model def serveropt_yogi(self, helper, pseudo_gradient, model_old, parameters): - """ Server side optimization, FedYogi. + """Server side optimization, FedYogi. :param helper: instance of helper class. :type helper: Helper @@ -214,14 +197,14 @@ def serveropt_yogi(self, helper, pseudo_gradient, model_old, parameters): self.v = helper.ones(pseudo_gradient, math.pow(tau, 2)) if not self.m: - self.m = helper.multiply(pseudo_gradient, [(1.0-beta1)]*len(pseudo_gradient)) + self.m = helper.multiply(pseudo_gradient, [(1.0 - beta1)] * len(pseudo_gradient)) else: - self.m = helper.add(self.m, pseudo_gradient, beta1, (1.0-beta1)) + self.m = helper.add(self.m, pseudo_gradient, beta1, (1.0 - beta1)) p = helper.power(pseudo_gradient, 2) s = helper.sign(helper.add(self.v, p, 1.0, -1.0)) s = helper.multiply(s, p) - self.v = helper.add(self.v, s, 1.0, -(1.0-beta2)) + self.v = helper.add(self.v, s, 1.0, -(1.0 - beta2)) sv = helper.add(helper.sqrt(self.v), helper.ones(self.v, tau)) t = helper.divide(self.m, sv) @@ -230,7 +213,7 @@ def serveropt_yogi(self, helper, pseudo_gradient, model_old, parameters): return model def serveropt_adagrad(self, helper, pseudo_gradient, model_old, parameters): - """ Server side optimization, FedAdam. + """Server side optimization, FedAdam. :param helper: instance of helper class. :type helper: Helper @@ -251,9 +234,9 @@ def serveropt_adagrad(self, helper, pseudo_gradient, model_old, parameters): self.v = helper.ones(pseudo_gradient, math.pow(tau, 2)) if not self.m: - self.m = helper.multiply(pseudo_gradient, [(1.0-beta1)]*len(pseudo_gradient)) + self.m = helper.multiply(pseudo_gradient, [(1.0 - beta1)] * len(pseudo_gradient)) else: - self.m = helper.add(self.m, pseudo_gradient, beta1, (1.0-beta1)) + self.m = helper.add(self.m, pseudo_gradient, beta1, (1.0 - beta1)) p = helper.power(pseudo_gradient, 2) self.v = helper.add(self.v, p, 1.0, 1.0) diff --git a/fedn/network/combiner/aggregators/functionproviderbase.py b/fedn/network/combiner/aggregators/functionproviderbase.py deleted file mode 100644 index ea53b7553..000000000 --- a/fedn/network/combiner/aggregators/functionproviderbase.py +++ /dev/null @@ -1,63 +0,0 @@ -from abc import ABC, ABCMeta, abstractmethod - -import numpy as np - - -class NamingEnforcementMeta(type): - """Metaclass that enforces the name 'FunctionProvider' for any class - inheriting from FunctionProviderBase. - """ - - def __init__(cls, name, bases, attrs): - if name != "FunctionProvider" and name != "FunctionProviderBase": # Allow the base class name - raise TypeError(f"Class {name} must be named 'FunctionProvider'") - super().__init__(name, bases, attrs) - - -class CombinedMeta(NamingEnforcementMeta, ABCMeta): - """A metaclass that combines ABCMeta and NamingEnforcementMeta to allow - the use of both abstract base classes and custom naming enforcement. - """ - - pass - - -class FunctionProviderBase(ABC, metaclass=CombinedMeta): - """Abstract base class that defines the structure for function providers. - Enforces the implementation of certain methods and provides shared functionality. - """ - - def __init__(self) -> None: - """Initialize the FunctionProviderBase class. This method can be overridden - by subclasses if initialization logic is required. - """ - pass - - @abstractmethod - def aggregate(self, parameters: list[list[np.ndarray]]) -> list[np.ndarray]: - """Aggregates a list of parameters from clients. - - Args: - ---- - parameters (list[list[np.ndarray]]): A list where each element is a list - of numpy arrays representing parameters from a client. - - Returns: - ------- - list[np.ndarray]: A list of numpy arrays representing the aggregated - parameters across all clients. - - """ - pass - - def get_model_metadata(self) -> dict: - """Returns metadata related to the model, which gets distributed to the clients. - The dictionary may only contain primitive types. - - - Returns - ------- - dict: A dictionary containing metadata information. - - """ - return {} diff --git a/fedn/network/combiner/aggregators/tests/test_fedavg.py b/fedn/network/combiner/aggregators/tests/test_fedavg.py index 55e5052b8..97013d7f4 100644 --- a/fedn/network/combiner/aggregators/tests/test_fedavg.py +++ b/fedn/network/combiner/aggregators/tests/test_fedavg.py @@ -17,8 +17,8 @@ def test_fedavg_init(self, *args, **kwargs): def test_fedavg_combine_models(self, *args, **kwargs): """Test the FedAvg aggregator combine_models method with mock classes and methods""" - aggregator = FedAvg("id", None, None, None, None) - aggregator.next_model_update = MagicMock(return_value=(None, None, None)) + aggregator = FedAvg("id", None) + aggregator.update_handler.get_model_updates = MagicMock(return_value=[(None, None, None)]) aggregator.server = MagicMock() data = {} diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index 78fe39026..cc1ba1ca7 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -476,7 +476,7 @@ def SetAggregator(self, control: fedn.ControlRequest, context): response.message = "Failed" return response - def SetFunctionProvider(self, control: fedn.ControlRequest, context): + def SetServerFunctions(self, control: fedn.ControlRequest, context): """Set a function provider. :param control: the control request @@ -486,11 +486,11 @@ def SetFunctionProvider(self, control: fedn.ControlRequest, context): :return: the control response :rtype: :class:`fedn.network.grpc.fedn_pb2.ControlResponse` """ - logger.debug("grpc.Combiner.SetFunctionProvider: Called") + logger.debug("grpc.Combiner.SetServerFunctions: Called") for parameter in control.parameter: - function_provider = parameter.value + server_functions = parameter.value - self.round_handler.function_provider_code = function_provider + self.round_handler.set_server_functions(server_functions) response = fedn.ControlResponse() response.message = "Success" @@ -686,7 +686,7 @@ def SendModelUpdate(self, request, context): :return: the response :rtype: :class:`fedn.network.grpc.fedn_pb2.Response` """ - self.round_handler.aggregator.on_model_update(request) + self.round_handler.update_handler.on_model_update(request) response = fedn.Response() response.response = "RECEIVED ModelUpdate {} from client {}".format(response, response.sender.name) diff --git a/fedn/network/combiner/hook_client.py b/fedn/network/combiner/hook_client.py deleted file mode 100644 index 8b8555ee9..000000000 --- a/fedn/network/combiner/hook_client.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -import os - -import grpc - -import fedn.network.grpc.fedn_pb2 as fedn -import fedn.network.grpc.fedn_pb2_grpc as rpc -from fedn.common.log_config import logger -from fedn.network.combiner.modelservice import load_model_from_BytesIO, serialize_model_to_BytesIO - - -class CombinerHookClient: - def __init__(self): - logger.info("Starting hook client") - self.hook_service_host = os.getenv("HOOK_SERVICE_HOST", "hook:12081") - self.channel = grpc.insecure_channel(self.hook_service_host) - self.stub = rpc.FunctionServiceStub(self.channel) - - def call_function_service(self, task, payload, meta={}): - if task == "setup" or task == "get_model_metadata": - request = fedn.FunctionRequest(task=task, payload_string=payload) - if task == "aggregate" or task == "store_parameters": - request = fedn.FunctionRequest(task=task, payload_bytes=payload, payload_string=json.dumps(meta)) - try: - response = self.stub.ExecuteFunction(request) - return response - except grpc.RpcError as e: - logger.info(f"RPC failed: {e}") - return None - - def set_function_provider(self, class_code): - if not isinstance(class_code, str): - raise TypeError("class_code must be of type string") - self.call_function_service("setup", class_code) - logger.info("Function provider code set.") - - def call_function(self, task, payload, helper, meta): - if task == "aggregate": - payload = serialize_model_to_BytesIO(payload, helper).getvalue() - response = self.call_function_service(task, payload, meta=meta) - return load_model_from_BytesIO(response.result_bytes, helper) - if task == "store_parameters": - payload = serialize_model_to_BytesIO(payload, helper).getvalue() - response = self.call_function_service(task, payload, meta=meta) - - def get_model_metadata(self): - response = self.call_function_service("get_model_metadata", "") - model_metadata_json = response.result_string - return json.loads(model_metadata_json) diff --git a/fedn/network/combiner/hooks/allowed_import.py b/fedn/network/combiner/hooks/allowed_import.py new file mode 100644 index 000000000..692790b12 --- /dev/null +++ b/fedn/network/combiner/hooks/allowed_import.py @@ -0,0 +1,7 @@ +import random # noqa: F401 +from typing import Dict, List, Tuple # noqa: F401 + +import numpy as np # noqa: F401 + +from fedn.common.log_config import logger # noqa: F401 +from fedn.network.combiner.hooks.serverfunctionsbase import ServerFunctionsBase # noqa: F401 diff --git a/fedn/network/combiner/hooks/hook_client.py b/fedn/network/combiner/hooks/hook_client.py new file mode 100644 index 000000000..77965981e --- /dev/null +++ b/fedn/network/combiner/hooks/hook_client.py @@ -0,0 +1,76 @@ +import json +import os +from io import BytesIO + +import grpc + +import fedn.network.grpc.fedn_pb2 as fedn +import fedn.network.grpc.fedn_pb2_grpc as rpc +from fedn.common.log_config import logger +from fedn.network.combiner.modelservice import bytesIO_request_generator, load_model_from_bytes, model_as_bytesIO +from fedn.network.combiner.updatehandler import UpdateHandler + +CHUNK_SIZE = 1024 * 1024 + + +class CombinerHookInterface: + def __init__(self): + logger.info("Starting hook client") + self.hook_service_host = os.getenv("HOOK_SERVICE_HOST", "hook:12081") + self.channel = grpc.insecure_channel(self.hook_service_host) + self.stub = rpc.FunctionServiceStub(self.channel) + + def provided_functions(self, server_functions: str): + """Communicates to hook container and asks which functions are available.""" + request = fedn.ProvidedFunctionsRequest(function_code=server_functions) + + response = self.stub.HandleProvidedFunctions(request) + return response.available_functions + + def client_config(self, global_model) -> dict: + """Communicates to hook container to get a client config.""" + request_function = fedn.ClientConfigRequest + args = {} + model = model_as_bytesIO(global_model) + response = self.stub.HandleClientConfig(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) + return json.loads(response.client_config) + + def client_selection(self, clients: list) -> list: + request = fedn.ClientSelectionRequest(client_ids=json.dumps(clients)) + response = self.stub.HandleClientSelection(request) + return json.loads(response.client_ids) + + def aggregate(self, previous_global, update_handler: UpdateHandler, helper, delete_models: bool): + """Aggregation call to the hook functions. + + Sends models in chunks, then asks for aggregation. + """ + data = {} + data["time_model_load"] = 0.0 + data["time_model_aggregation"] = 0.0 + # send previous global + request_function = fedn.AggregationRequest + args = {"id": "global_model", "aggregate": False} + model = model_as_bytesIO(previous_global) + self.stub.HandleAggregation(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) + # send client models and metadata + updates = update_handler.get_model_updates() + for update in updates: + model, metadata = update_handler.load_model_update(update, helper) + # send metadata + client_id = update.sender.client_id + request = fedn.ClientMetaRequest(metadata=metadata, client_id=client_id) + self.stub.HandleMetadata(request) + # send client model + model = model_as_bytesIO(model) + args = {"id": client_id, "aggregate": False} + request_function = fedn.AggregationRequest + self.stub.HandleAggregation(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) + if delete_models: + # delete model from disk + update_handler.delete_model(model_update=update) + # ask for aggregation + request = fedn.AggregationRequest(data=None, client_id="", aggregate=True) + response = self.stub.HandleAggregation(request) + data["nr_aggregated_models"] = len(updates) + return load_model_from_bytes(response.data, helper), data diff --git a/fedn/network/combiner/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py index 98f4d9f25..5559e8551 100644 --- a/fedn/network/combiner/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -1,13 +1,17 @@ +import inspect import json from concurrent import futures +from io import BytesIO import grpc import fedn.network.grpc.fedn_pb2 as fedn import fedn.network.grpc.fedn_pb2_grpc as rpc from fedn.common.log_config import logger -from fedn.network.combiner.hooks.safe_builtins import safe_builtins -from fedn.network.combiner.modelservice import load_model_from_BytesIO, serialize_model_to_BytesIO + +# imports for user code +from fedn.network.combiner.hooks.allowed_import import Dict, List, ServerFunctionsBase, Tuple, np, random # noqa: F401 +from fedn.network.combiner.modelservice import bytesIO_request_generator, load_model_from_bytes, model_as_bytesIO from fedn.utils.helpers.plugins.numpyhelper import Helper CHUNK_SIZE = 1024 * 1024 @@ -17,74 +21,121 @@ class FunctionServiceServicer(rpc.FunctionServiceServicer): def __init__(self) -> None: super().__init__() - self.safe_builtins = safe_builtins - self.globals_dict = { - "__builtins__": self.safe_builtins, - } self.helper = Helper() - self.client_results = [] + self.server_functions: ServerFunctionsBase = None + self.client_updates = {} + + def HandleClientConfig(self, request_iterator, context): + logger.info("Received client config request.") + model = self.unpack_model(request_iterator) + client_config = self.server_functions.client_config(global_model=model) + logger.info(f"Client config response: {client_config}") + return fedn.ClientConfigResponse(client_config=json.dumps(client_config)) + + def HandleClientSelection(self, request, context): + logger.info("Received client selection request.") + client_ids = json.loads(request.client_ids) + client_ids = self.server_functions.client_selection(client_ids) + logger.info(f"Clients selected: {client_ids}") + return fedn.ClientSelectionResponse(client_ids=json.dumps(client_ids)) + + def HandleMetadata(self, request, context): + client_id = request.client_id + metadata = json.loads(request.metadata) + self.client_updates[client_id] = self.client_updates.get(client_id, []) + [metadata] + return fedn.ClientMetaResponse(status="Metadata stored") + + def HandleAggregation(self, request_iterator, context): + # check what type of request + for request in request_iterator: + if request.aggregate: + logger.info("Received aggregation request.") + aggregated_model = self.server_functions.aggregate(self.previous_global, self.client_updates) + aggregated_model = model_as_bytesIO(aggregated_model) + request_function = fedn.AggregationResponse + logger.info("Returning aggregate model.") + return bytesIO_request_generator(mdl=aggregated_model, request_function=request_function, args={}) + client_id = request.client_id + break + logger.info(f"Received request to store model originating from: {client_id}") + model = self.unpack_model(request_iterator) + + if client_id == "global_model": + self.previous_global = model + else: + self.client_updates[client_id] = [model] + self.client_updates.get(client_id, []) + return fedn.AggregationResponse(data=None) + + def HandleProvidedFunctions(self, request, context): + """Handles the 'provided_functions' request. Sends back which functions are available.""" + logger.info("Receieved provided functions request.") + server_functions_code = request.function_code + if self.server_functions is None: + # this will create a new user defined instance of the ServerFunctions class. + try: + namespace = {} + exec(server_functions_code, globals(), namespace) # noqa: S102 + exec("server_functions = ServerFunctions()", globals(), namespace) # noqa: S102 + self.server_functions = namespace.get("server_functions") + except Exception as e: + logger.error(f"Exec failed with error: {str(e)}") + implemented_functions = {} + # if crashed or not returning None we assume function is implemented + # check if aggregation is available + try: + ret = self.server_functions.aggregate(0, 0) + logger.info(f"ret : {ret}") + if ret is None: + implemented_functions["aggregate"] = False + else: + implemented_functions["aggregate"] = True + except Exception: + implemented_functions["aggregate"] = True + # check if client_config is available + try: + ret = self.server_functions.client_config(0) + if ret is None: + implemented_functions["client_config"] = False + else: + implemented_functions["client_config"] = True + except Exception: + implemented_functions["client_config"] = True + # check if client_selection is available + try: + ret = self.server_functions.client_selection(0) + if ret is None: + implemented_functions["client_selection"] = False + else: + implemented_functions["client_selection"] = False + except Exception: + implemented_functions["client_selection"] = False + logger.info(f"Provided function: {implemented_functions}") + return fedn.ProvidedFunctionsResponse(available_functions=implemented_functions) + + def unpack_model(self, request_iterator): + """Unpack the incoming model sent in chunks from the request iterator. - def ExecuteFunction(self, request, context): - # Compile the function code + :param request_iterator: A streaming iterator from the gRPC service. + :return: The reconstructed model parameters. + """ + model_buffer = BytesIO() try: - if request.task == "setup": - logger.info("Adding function provider.") - self.init_hook_object(request.payload_string) - return fedn.FunctionResponse(result_string="Instansiated hook functions.") - if request.task == "store_parameters": - logger.info("Executing aggregate function.") - parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) - self.client_results.append((parameters, json.loads(request.payload_string))) - return fedn.FunctionResponse(result_string="Stored parameters") - if request.task == "aggregate": - logger.info("Executing aggregate function.") - parameters = load_model_from_BytesIO(request.payload_bytes, self.helper) - self.client_results.append((parameters, json.loads(request.payload_string))) - result = self.execute_function_code() - result_bytes = serialize_model_to_BytesIO(result, self.helper).getvalue() - return fedn.FunctionResponse(result_bytes=result_bytes) - if request.task == "get_model_metadata": - model_metadata = self.get_model_metadata() - json_model_metadata = json.dumps(model_metadata, indent=4) - return fedn.FunctionResponse(result_string=json_model_metadata) + for request in request_iterator: + if request.data: + model_buffer.write(request.data) + except MemoryError as e: + print(f"Memory error occured when loading model, reach out to the FEDn team if you need a solution to this. {e}") + raise except Exception as e: - logger.error(f"Error executing task {request.task}: {e}", exc_info=True) - context.set_details(f"Error: {e}") - context.set_code(grpc.StatusCode.INTERNAL) - return fedn.FunctionResponse(result_string=f"Error executing task: {e}") - - def init_hook_object(self, class_code): - # Prepare the globals dictionary with restricted builtins - - # Compile and execute the class code - exec(class_code, self.globals_dict) # noqa: S102 - # Instantiate the object within restricted scope - instance_code = """ -function_provider = FunctionProvider() -""" - # Compile and execute the instance code - exec(instance_code, self.globals_dict) # noqa: S102 - - def execute_function_code(self): - if not hasattr(self, "globals_dict"): - raise AttributeError("Function provider code has not been provided.") - self.globals_dict["client_results"] = self.client_results - instance_code = """ -res = function_provider.aggregate(client_results) -""" - exec(instance_code, self.globals_dict) # noqa: S102 - self.client_results = [] - return self.globals_dict["res"] - - def get_model_metadata(self): - if not hasattr(self, "globals_dict"): - raise AttributeError("Function provider code has not been provided.") - instance_code = """ -model_metadata = function_provider.get_model_metadata() -""" - exec(instance_code, self.globals_dict) # noqa: S102 - return self.globals_dict["model_metadata"] + print(f"Exception occured during model loading: {e}") + raise + + model_buffer.seek(0) + + model_bytes = model_buffer.getvalue() + + return load_model_from_bytes(model_bytes, self.helper) def serve(): diff --git a/fedn/network/combiner/hooks/safe_builtins.py b/fedn/network/combiner/hooks/safe_builtins.py deleted file mode 100644 index 8c989aaf2..000000000 --- a/fedn/network/combiner/hooks/safe_builtins.py +++ /dev/null @@ -1,72 +0,0 @@ -import builtins - -safe_builtins = { - "abs": abs, - "all": all, - "any": any, - "ascii": ascii, - "bin": bin, - "bool": bool, - "bytearray": bytearray, - "bytes": bytes, - "callable": callable, - "chr": chr, - "complex": complex, - "dict": dict, - "divmod": divmod, - "enumerate": enumerate, - "filter": filter, - "float": float, - "format": format, - "frozenset": frozenset, - "getattr": getattr, - "hasattr": hasattr, - "hash": hash, - "hex": hex, - "id": id, - "int": int, - "isinstance": isinstance, - "issubclass": issubclass, - "iter": iter, - "len": len, - "list": list, - "map": map, - "max": max, - "min": min, - "next": next, - "object": object, - "oct": oct, - "ord": ord, - "pow": pow, - "print": print, - "property": property, - "range": range, - "repr": repr, - "reversed": reversed, - "round": round, - "set": set, - "slice": slice, - "sorted": sorted, - "staticmethod": staticmethod, - "str": str, - "sum": sum, - "super": super, - "tuple": tuple, - "type": type, - "zip": zip, - "__build_class__": builtins.__build_class__, - "__name__": builtins.__name__, - "__import__": builtins.__import__, - "BaseException": BaseException, - "Exception": Exception, - "ArithmeticError": ArithmeticError, - "BufferError": BufferError, - "LookupError": LookupError, - "math": __import__("math"), - "itertools": __import__("itertools"), - "functools": __import__("functools"), - "operator": __import__("operator"), - "collections": __import__("collections"), - "re": __import__("re"), - "datetime": __import__("datetime"), -} diff --git a/fedn/network/combiner/hooks/serverfunctionsbase.py b/fedn/network/combiner/hooks/serverfunctionsbase.py new file mode 100644 index 000000000..b0e4efb67 --- /dev/null +++ b/fedn/network/combiner/hooks/serverfunctionsbase.py @@ -0,0 +1,74 @@ +from abc import ABC +from typing import Dict, List, Tuple + +import numpy as np + + +class ServerFunctionsBase(ABC): + """Base class that defines the structure for the Server Functions. Override these functions + to add to the server workflow. + + Available packages are: + numpy as np, and most common python built in packages. + """ + + def __init__(self) -> None: + """Initialize the ServerFunctionsBase class. This method can be overridden + by subclasses if initialization logic is required. + """ + pass + + def aggregate(self, previous_global: list[np.ndarray], client_updates: Dict[str, Tuple[List[np.ndarray], dict]]) -> list[np.ndarray]: + """Aggregates a list of parameters from clients. + + Args: + ---- + previous_global (list[np.ndarray]): A list of parameters representing the global + model from the previous round. + + client_updates (list[list[np.ndarray]]): A dictionary where the key is client ID, + pointing to a tuple with the first element being client parameter and second element + being the clients metadata. + + Returns: + ------- + list[np.ndarray]: A list of numpy arrays representing the aggregated + parameters across all clients. + + """ + pass + + def client_config(self, global_model: list[np.ndarray]) -> dict: + """Returns metadata related to the model, which gets distributed to the clients. + The dictionary may only contain primitive types. + + Args: + ---- + global_model (list[np.ndarray]): A list of parameters representing the global + model for the upcomming round. + + Returns: + ------- + dict: A dictionary containing metadata information, supporting only primitive python types. + + """ + pass + + def client_selection(self, client_ids: list[str]) -> list: + """Returns a list of client_id's of which clients to be used for the next training request. + + Args: + ---- + client_ids (list[str]): A list of client_ids for all connected clients. + + Returns: + ------- + list[str]: A list of client ids for which clients should be chosen for the next training round. + + """ + pass + + +# base implementation +class ServerFunctions(ServerFunctionsBase): + pass diff --git a/fedn/network/combiner/interfaces.py b/fedn/network/combiner/interfaces.py index 32c8dbf41..f9c77d2f0 100644 --- a/fedn/network/combiner/interfaces.py +++ b/fedn/network/combiner/interfaces.py @@ -203,7 +203,7 @@ def set_aggregator(self, aggregator): else: raise - def set_function_provider(self, function_provider): + def set_server_functions(self, server_functions): """Set the function provider module. :param function provider: Stringified function provider code. @@ -214,11 +214,11 @@ def set_function_provider(self, function_provider): request = fedn.ControlRequest() p = request.parameter.add() - p.key = "function_provider" - p.value = function_provider + p.key = "server_functions" + p.value = server_functions try: - control.SetFunctionProvider(request) + control.SetServerFunctions(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.UNAVAILABLE: raise CombinerUnavailableError diff --git a/fedn/network/combiner/modelservice.py b/fedn/network/combiner/modelservice.py index b5e7bff73..7571182af 100644 --- a/fedn/network/combiner/modelservice.py +++ b/fedn/network/combiner/modelservice.py @@ -29,7 +29,32 @@ def upload_request_generator(mdl, id): break +def bytesIO_request_generator(mdl, request_function, args): + """Generator function for model upload requests. + + :param mdl: The model update object. + :type mdl: BytesIO + :param request_function: Function for sending requests. + :type request_function: Function + :param args: request arguments, excluding data argument. + :type args: dict + :return: Yields grpc request for streaming. + :rtype: grpc request generator. + """ + while True: + b = mdl.read(CHUNK_SIZE) + if b: + result = request_function(data=b, **args) + else: + result = request_function(data=None, **args) + yield result + if not b: + break + + def model_as_bytesIO(model): + if isinstance(model, list): + serialize_model_to_BytesIO(model) if not isinstance(model, BytesIO): bt = BytesIO() @@ -51,10 +76,10 @@ def get_tmp_path(): return path -def load_model_from_BytesIO(model_bytesio, helper): - """Load a model from a BytesIO object. - :param model_bytesio: A BytesIO object containing the model. - :type model_bytesio: :class:`io.BytesIO` +def load_model_from_bytes(model_bytes, helper): + """Load a model from a bytes object. + :param model_bytesio: A bytes object containing the model. + :type model_bytes: :class:`bytes` :param helper: The helper object for the model. :type helper: :class:`fedn.utils.helperbase.HelperBase` :return: The model object. @@ -62,7 +87,7 @@ def load_model_from_BytesIO(model_bytesio, helper): """ path = get_tmp_path() with open(path, "wb") as fh: - fh.write(model_bytesio) + fh.write(model_bytes) fh.flush() model = helper.load(path) os.unlink(path) diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index c950c5ef7..fd4119a28 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -1,15 +1,18 @@ import ast +import inspect import queue import random -import sys import time import uuid from typing import TypedDict from fedn.common.log_config import logger from fedn.network.combiner.aggregators.aggregatorbase import get_aggregator -from fedn.network.combiner.hook_client import CombinerHookClient -from fedn.network.combiner.modelservice import load_model_from_BytesIO, serialize_model_to_BytesIO +from fedn.network.combiner.hooks.hook_client import CombinerHookInterface +from fedn.network.combiner.hooks.serverfunctionsbase import ServerFunctions +from fedn.network.combiner.modelservice import ModelService, serialize_model_to_BytesIO +from fedn.network.combiner.updatehandler import UpdateHandler +from fedn.network.storage.s3.repository import Repository from fedn.utils.helpers.helpers import get_helper from fedn.utils.parameters import Parameters @@ -46,6 +49,8 @@ class RoundConfig(TypedDict): :type helper_type: str :param aggregator: The aggregator type. :type aggregator: str + :param client_config: Configs that are distributed to clients. + :type client_config: dict """ _job_id: str @@ -65,10 +70,6 @@ class RoundConfig(TypedDict): aggregator: str -class ModelUpdateError(Exception): - pass - - class RoundHandler: """Round handler. @@ -85,21 +86,20 @@ class RoundHandler: :type modelservice: class: `fedn.network.combiner.modelservice.ModelService` """ - def __init__(self, storage, server, modelservice): + def __init__(self, storage: Repository, server, modelservice: ModelService): """Initialize the RoundHandler.""" self.round_configs = queue.Queue() self.storage = storage self.server = server self.modelservice = modelservice + self.server_functions = inspect.getsource(ServerFunctions) + self.update_handler = UpdateHandler(modelservice=modelservice) def set_aggregator(self, aggregator): - self.aggregator = get_aggregator(aggregator, self.storage, self.server, self.modelservice, self) + self.aggregator = get_aggregator(aggregator, self.update_handler) - def set_function_provider(self): - if not hasattr(self, "function_provider_code") or self.function_provider_code is None: - raise Exception("Custom function provider code need to be set.") - self.combiner_hook_client = CombinerHookClient() - self.combiner_hook_client.set_function_provider(self.function_provider_code) + def set_server_functions(self, server_functions: str): + self.server_functions = server_functions def push_round_config(self, round_config: RoundConfig) -> str: """Add a round_config (job description) to the inbox. @@ -117,76 +117,7 @@ def push_round_config(self, round_config: RoundConfig) -> str: raise return round_config["_job_id"] - def load_model_update(self, helper, model_id): - """Load model update with id model_id into its memory representation. - - :param helper: An instance of :class: `fedn.utils.helpers.helpers.HelperBase` - :type helper: class: `fedn.utils.helpers.helpers.HelperBase` - :param model_id: The ID of the model update, UUID in str format - :type model_id: str - """ - model_str = self.load_model_update_str(model_id) - if model_str: - try: - model = load_model_from_BytesIO(model_str.getbuffer(), helper) - except IOError: - logger.warning("AGGREGATOR({}): Failed to load model!".format(self.name)) - else: - raise ModelUpdateError("Failed to load model.") - - return model - - def load_model_update_str(self, model_id, retry=3): - """Load model update object and return it as BytesIO. - - :param model_id: The ID of the model - :type model_id: str - :param retry: number of times retrying load model update, defaults to 3 - :type retry: int, optional - :return: Updated model - :rtype: class: `io.BytesIO` - """ - # Try reading model update from local disk/combiner memory - model_str = self.modelservice.temp_model_storage.get(model_id) - # And if we cannot access that, try downloading from the server - if model_str is None: - model_str = self.modelservice.get_model(model_id) - # TODO: use retrying library - tries = 0 - while tries < retry: - tries += 1 - if not model_str or sys.getsizeof(model_str) == 80: - logger.warning("Model download failed. retrying") - time.sleep(1) - model_str = self.modelservice.get_model(model_id) - - return model_str - - def waitforit(self, config, buffer_size=100, polling_interval=0.1): - """Defines the policy for how long the server should wait before starting to aggregate models. - - The policy is as follows: - 1. Wait a maximum of time_window time until the round times out. - 2. Terminate if a preset number of model updates (buffer_size) are in the queue. - - :param config: The round config object - :type config: dict - :param buffer_size: The number of model updates to wait for before starting aggregation, defaults to 100 - :type buffer_size: int, optional - :param polling_interval: The polling interval, defaults to 0.1 - :type polling_interval: float, optional - """ - time_window = float(config["round_timeout"]) - - tt = 0.0 - while tt < time_window: - if self.aggregator.model_updates.qsize() >= buffer_size: - break - - time.sleep(polling_interval) - tt += polling_interval - - def _training_round(self, config, clients): + def _training_round(self, config, clients, provided_functions): """Send model update requests to clients and aggregate results. :param config: The round config object (passed to the client). @@ -206,6 +137,10 @@ def _training_round(self, config, clients): session_id = config["session_id"] model_id = config["model_id"] + if provided_functions["client_config"]: + global_model_bytes = self.modelservice.temp_model_storage.get(model_id) + client_config = CombinerHookInterface().client_config(global_model_bytes) + config["client_config"] = client_config # Request model updates from all active clients. self.server.request_model_update(session_id=session_id, model_id=model_id, config=config, clients=clients) @@ -216,12 +151,10 @@ def _training_round(self, config, clients): buffer_size = int(config["buffer_size"]) # Wait / block until the round termination policy has been met. - self.waitforit(config, buffer_size=buffer_size) - + self.update_handler.waitforit(config, buffer_size=buffer_size) tic = time.time() model = None data = None - try: helper = get_helper(config["helper_type"]) logger.info("Config delete_models_storage: {}".format(config["delete_models_storage"])) @@ -235,11 +168,16 @@ def _training_round(self, config, clients): parameters = Parameters(dict_parameters) else: parameters = None - - model, data = self.aggregator.combine_models(helper=helper, delete_models=delete_models, parameters=parameters) + if provided_functions["aggregate"]: + previous_model_bytes = self.modelservice.temp_model_storage.get(model_id) + model, data = CombinerHookInterface().aggregate(previous_model_bytes, self.update_handler, helper, delete_models=delete_models) + else: + model, data = self.aggregator.combine_models(helper=helper, delete_models=delete_models, parameters=parameters) except Exception as e: logger.warning("AGGREGATION FAILED AT COMBINER! {}".format(e)) + self.update_handler.flush() + meta["time_combination"] = time.time() - tic meta["aggregation_time"] = data return model, meta @@ -383,8 +321,13 @@ def execute_training_round(self, config): # Download model to update and set in temp storage. self.stage_model(config["model_id"]) - clients = self._assign_round_clients(self.server.max_clients) - model, meta = self._training_round(config, clients) + provided_functions = CombinerHookInterface().provided_functions(self.server_functions) + + if provided_functions["client_selection"]: + clients = CombinerHookInterface().client_selection(clients=self.server.get_active_trainers()) + else: + clients = self._assign_round_clients(self.server.max_clients) + model, meta = self._training_round(config, clients, provided_functions) data["data"] = meta if model is None: @@ -423,7 +366,6 @@ def run(self, polling_interval=1.0): if ready: if round_config["task"] == "training": tic = time.time() - round_config["model_metadata"] = self.aggregator.get_model_metadata() round_meta = self.execute_training_round(round_config) round_meta["time_exec_training"] = time.time() - tic round_meta["status"] = "Success" diff --git a/fedn/network/combiner/updatehandler.py b/fedn/network/combiner/updatehandler.py new file mode 100644 index 000000000..d2f395688 --- /dev/null +++ b/fedn/network/combiner/updatehandler.py @@ -0,0 +1,204 @@ +import json +import sys +import time +import traceback + +from fedn.common.log_config import logger +from fedn.network.combiner.modelservice import ModelService, load_model_from_bytes + + +class ModelUpdateError(Exception): + pass + + +class UpdateHandler: + """Update handler. + + Responsible for receiving, loading and supplying client model updates. + + :param modelservice: A handle to the model service :class: `fedn.network.combiner.modelservice.ModelService` + :type modelservice: class: `fedn.network.combiner.modelservice.ModelService` + """ + + def __init__(self, modelservice: ModelService) -> None: + self.model_updates = [] + self.modelservice = modelservice + + self.model_id_to_model_data = {} + + def delete_model(self, model_update): + self.modelservice.temp_model_storage.delete(model_update.model_update_id) + logger.info("UPDATE HANDLER: Deleted model update {} from storage.".format(model_update.model_update_id)) + + def get_model_updates(self): + return self.model_updates + + def on_model_update(self, model_update): + """Callback when a new client model update is recieved. + + Performs (optional) validation and pre-processing, + and then puts the update id on the aggregation queue. + Override in subclass as needed. + + :param model_update: fedn.network.grpc.fedn.proto.ModelUpdate + :type model_id: str + """ + try: + logger.info("UPDATE HANDLER: callback received model update {}".format(model_update.model_update_id)) + + # Validate the update and metadata + valid_update = self._validate_model_update(model_update) + if valid_update: + # Push the model update to the processing queue + self.model_updates.append(model_update) + else: + logger.warning("UPDATE HANDLER: Invalid model update, skipping.") + except Exception as e: + tb = traceback.format_exc() + logger.error("UPDATE HANDLER: failed to receive model update: {}".format(e)) + logger.error(tb) + pass + + def _validate_model_update(self, model_update): + """Validate the model update. + + :param model_update: A ModelUpdate message. + :type model_update: object + :return: True if the model update is valid, False otherwise. + :rtype: bool + """ + try: + data = json.loads(model_update.meta)["training_metadata"] + _ = data["num_examples"] + except KeyError: + tb = traceback.format_exc() + logger.error("AGGREGATOR({}): Invalid model update, missing metadata.".format(self.name)) + logger.error(tb) + return False + return True + + def load_model_update(self, model_update, helper): + """Load the memory representation of the model update. + + Load the model update paramters and the + associate metadata into memory. + + :param model_update: The model update. + :type model_update: fedn.network.grpc.fedn.proto.ModelUpdate + :param helper: A helper object. + :type helper: fedn.utils.helpers.helperbase.Helper + :return: A tuple of (parameters, metadata) + :rtype: tuple + """ + model_id = model_update.model_update_id + model = self.load_model(helper, model_id) + # Get relevant metadata + metadata = json.loads(model_update.meta) + if "config" in metadata.keys(): + # Used in Python client + config = json.loads(metadata["config"]) + else: + # Used in C++ client + config = json.loads(model_update.config) + training_metadata = metadata["training_metadata"] + training_metadata["round_id"] = config["round_id"] + + return model, training_metadata + + def load_model_update_byte(self, model_update): + """Load the memory representation of the model update. + + Load the model update paramters and the + associate metadata into memory. + + :param model_update: The model update. + :type model_update: fedn.network.grpc.fedn.proto.ModelUpdate + :return: A tuple of parameters(bytes), metadata + :rtype: tuple + """ + model_id = model_update.model_update_id + model = self._load_model_update_str(model_id).getbuffer() + # Get relevant metadata + metadata = json.loads(model_update.meta) + if "config" in metadata.keys(): + # Used in Python client + config = json.loads(metadata["config"]) + else: + # Used in C++ client + config = json.loads(model_update.config) + training_metadata = metadata["training_metadata"] + training_metadata["round_id"] = config["round_id"] + + return model, training_metadata + + def load_model(self, helper, model_id): + """Load model update with id model_id into its memory representation. + + :param helper: An instance of :class: `fedn.utils.helpers.helpers.HelperBase` + :type helper: class: `fedn.utils.helpers.helpers.HelperBase` + :param model_id: The ID of the model update, UUID in str format + :type model_id: str + """ + model_str = self._load_model_update_str(model_id) + if model_str: + try: + model = load_model_from_bytes(model_str.getbuffer(), helper) + except IOError: + logger.warning("AGGREGATOR({}): Failed to load model!".format(self.name)) + else: + raise ModelUpdateError("Failed to load model.") + + return model + + def _load_model_update_str(self, model_id, retry=3): + """Load model update object and return it as BytesIO. + + :param model_id: The ID of the model + :type model_id: str + :param retry: number of times retrying load model update, defaults to 3 + :type retry: int, optional + :return: Updated model + :rtype: class: `io.BytesIO` + """ + # Try reading model update from local disk/combiner memory + model_str = self.modelservice.temp_model_storage.get(model_id) + # And if we cannot access that, try downloading from the server + if model_str is None: + model_str = self.modelservice.get_model(model_id) + # TODO: use retrying library + tries = 0 + while tries < retry: + tries += 1 + if not model_str or sys.getsizeof(model_str) == 80: + logger.warning("Model download failed. retrying") + time.sleep(1) + model_str = self.modelservice.get_model(model_id) + + return model_str + + def waitforit(self, config, buffer_size=100, polling_interval=0.1): + """Defines the policy for how long the server should wait before starting to aggregate models. + + The policy is as follows: + 1. Wait a maximum of time_window time until the round times out. + 2. Terminate if a preset number of model updates (buffer_size) are in the queue. + + :param config: The round config object + :type config: dict + :param buffer_size: The number of model updates to wait for before starting aggregation, defaults to 100 + :type buffer_size: int, optional + :param polling_interval: The polling interval, defaults to 0.1 + :type polling_interval: float, optional + """ + time_window = float(config["round_timeout"]) + + tt = 0.0 + while tt < time_window: + if len(self.model_updates) >= buffer_size: + break + + time.sleep(polling_interval) + tt += polling_interval + + def flush(self): + self.model_updates = [] diff --git a/fedn/network/controller/control.py b/fedn/network/controller/control.py index 4386edd17..6d88458cc 100644 --- a/fedn/network/controller/control.py +++ b/fedn/network/controller/control.py @@ -7,7 +7,7 @@ from fedn.common.log_config import logger from fedn.network.combiner.interfaces import CombinerUnavailableError -from fedn.network.combiner.modelservice import load_model_from_BytesIO +from fedn.network.combiner.modelservice import load_model_from_bytes from fedn.network.combiner.roundhandler import RoundConfig from fedn.network.controller.controlbase import ControlBase from fedn.network.state import ReducerState @@ -129,8 +129,8 @@ def start_session(self, session_id: str, rounds: int, round_timeout: int) -> Non for combiner in self.network.get_combiners(): combiner.set_aggregator(aggregator) - if session_config["function_provider"] is not None: - combiner.set_function_provider(session_config["function_provider"]) + if session_config["server_functions"] is not None: + combiner.set_server_functions(session_config["server_functions"]) self.set_session_status(session_id, "Started") @@ -186,8 +186,8 @@ def session(self, config: RoundConfig) -> None: for combiner in self.network.get_combiners(): combiner.set_aggregator(config["aggregator"]) - if config["function_provider"] is not None: - combiner.set_function_provider(config["function_provider"]) + if config["server_functions"] is not None: + combiner.set_server_functions(config["server_functions"]) self.set_session_status(config["session_id"], "Started") # Execute the rounds in this session @@ -426,14 +426,14 @@ def reduce(self, combiners): try: tic = time.time() helper = self.get_helper() - model_next = load_model_from_BytesIO(data, helper) + model_next = load_model_from_bytes(data, helper) meta["time_load_model"] += time.time() - tic tic = time.time() model = helper.increment_average(model, model_next, 1.0, i) meta["time_aggregate_model"] += time.time() - tic except Exception: tic = time.time() - model = load_model_from_BytesIO(data, helper) + model = load_model_from_bytes(data, helper) meta["time_aggregate_model"] += time.time() - tic i = i + 1 diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index 92ac7e259..f052858e3 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -195,7 +195,7 @@ service Control { rpc Stop(ControlRequest) returns (ControlResponse); rpc FlushAggregationQueue(ControlRequest) returns (ControlResponse); rpc SetAggregator(ControlRequest) returns (ControlResponse); - rpc SetFunctionProvider(ControlRequest) returns (ControlResponse); + rpc SetServerFunctions(ControlRequest) returns (ControlResponse); } service Reducer { @@ -245,19 +245,53 @@ service Combiner { } -message FunctionRequest { - string task = 1; - string payload_string = 2; - bytes payload_bytes = 3; +message ProvidedFunctionsRequest { + string function_code = 1; } -message FunctionResponse { - oneof result { - string result_string = 2; - bytes result_bytes = 3; - } +message ProvidedFunctionsResponse { + map available_functions = 1; } -service FunctionService { - rpc ExecuteFunction(FunctionRequest) returns (FunctionResponse); +message ClientConfigRequest { + bytes data = 1; +} + +message ClientConfigResponse { + string client_config = 1; +} + +message ClientSelectionRequest { + string client_ids = 1; +} + +message ClientSelectionResponse { + string client_ids = 1; +} + +message ClientMetaRequest { + string metadata = 1; + string client_id = 2; } + +message ClientMetaResponse { + string status = 1; +} + +message AggregationRequest { + bytes data = 1; + string client_id = 2; + bool aggregate = 3; +} + +message AggregationResponse { + bytes data = 1; +} + +service FunctionService { + rpc HandleProvidedFunctions(ProvidedFunctionsRequest) returns (ProvidedFunctionsResponse); + rpc HandleClientConfig (stream ClientConfigRequest) returns (ClientConfigResponse); + rpc HandleClientSelection (ClientSelectionRequest) returns (ClientSelectionResponse); + rpc HandleMetadata (ClientMetaRequest) returns (ClientMetaResponse); + rpc HandleAggregation (stream AggregationRequest) returns (stream AggregationResponse); +} \ No newline at end of file diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index eca6f5965..6a2e94c60 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -15,25 +15,27 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"N\n\x0f\x46unctionRequest\x12\x0c\n\x04task\x18\x01 \x01(\t\x12\x16\n\x0epayload_string\x18\x02 \x01(\t\x12\x15\n\rpayload_bytes\x18\x03 \x01(\x0c\"M\n\x10\x46unctionResponse\x12\x17\n\rresult_string\x18\x02 \x01(\tH\x00\x12\x16\n\x0cresult_bytes\x18\x03 \x01(\x0cH\x00\x42\x08\n\x06result*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbc\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x42\n\x13SetFunctionProvider\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2S\n\x0f\x46unctionService\x12@\n\x0f\x45xecuteFunction\x12\x15.fedn.FunctionRequest\x1a\x16.fedn.FunctionResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"1\n\x18ProvidedFunctionsRequest\x12\x15\n\rfunction_code\x18\x01 \x01(\t\"\xac\x01\n\x19ProvidedFunctionsResponse\x12T\n\x13\x61vailable_functions\x18\x01 \x03(\x0b\x32\x37.fedn.ProvidedFunctionsResponse.AvailableFunctionsEntry\x1a\x39\n\x17\x41vailableFunctionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"#\n\x13\x43lientConfigRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"-\n\x14\x43lientConfigResponse\x12\x15\n\rclient_config\x18\x01 \x01(\t\",\n\x16\x43lientSelectionRequest\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"-\n\x17\x43lientSelectionResponse\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"8\n\x11\x43lientMetaRequest\x12\x10\n\x08metadata\x18\x01 \x01(\t\x12\x11\n\tclient_id\x18\x02 \x01(\t\"$\n\x12\x43lientMetaResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"H\n\x12\x41ggregationRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x11\n\tclient_id\x18\x02 \x01(\t\x12\x11\n\taggregate\x18\x03 \x01(\x08\"#\n\x13\x41ggregationResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbb\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x41\n\x12SetServerFunctions\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2\xa5\x03\n\x0f\x46unctionService\x12Z\n\x17HandleProvidedFunctions\x12\x1e.fedn.ProvidedFunctionsRequest\x1a\x1f.fedn.ProvidedFunctionsResponse\x12M\n\x12HandleClientConfig\x12\x19.fedn.ClientConfigRequest\x1a\x1a.fedn.ClientConfigResponse(\x01\x12T\n\x15HandleClientSelection\x12\x1c.fedn.ClientSelectionRequest\x1a\x1d.fedn.ClientSelectionResponse\x12\x43\n\x0eHandleMetadata\x12\x17.fedn.ClientMetaRequest\x1a\x18.fedn.ClientMetaResponse\x12L\n\x11HandleAggregation\x12\x18.fedn.AggregationRequest\x1a\x19.fedn.AggregationResponse(\x01\x30\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'fedn.network.grpc.fedn_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_STATUSTYPE']._serialized_start=2491 - _globals['_STATUSTYPE']._serialized_end=2623 - _globals['_QUEUE']._serialized_start=2625 - _globals['_QUEUE']._serialized_end=2661 - _globals['_MODELSTATUS']._serialized_start=2663 - _globals['_MODELSTATUS']._serialized_end=2746 - _globals['_ROLE']._serialized_start=2748 - _globals['_ROLE']._serialized_end=2804 - _globals['_COMMAND']._serialized_start=2806 - _globals['_COMMAND']._serialized_end=2880 - _globals['_CONNECTIONSTATUS']._serialized_start=2882 - _globals['_CONNECTIONSTATUS']._serialized_end=2955 + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._options = None + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_options = b'8\001' + _globals['_STATUSTYPE']._serialized_start=2942 + _globals['_STATUSTYPE']._serialized_end=3074 + _globals['_QUEUE']._serialized_start=3076 + _globals['_QUEUE']._serialized_end=3112 + _globals['_MODELSTATUS']._serialized_start=3114 + _globals['_MODELSTATUS']._serialized_end=3197 + _globals['_ROLE']._serialized_start=3199 + _globals['_ROLE']._serialized_end=3255 + _globals['_COMMAND']._serialized_start=3257 + _globals['_COMMAND']._serialized_end=3331 + _globals['_CONNECTIONSTATUS']._serialized_start=3333 + _globals['_CONNECTIONSTATUS']._serialized_end=3406 _globals['_RESPONSE']._serialized_start=71 _globals['_RESPONSE']._serialized_end=129 _globals['_STATUS']._serialized_start=132 @@ -78,20 +80,38 @@ _globals['_CONNECTIONREQUEST']._serialized_end=2267 _globals['_CONNECTIONRESPONSE']._serialized_start=2269 _globals['_CONNECTIONRESPONSE']._serialized_end=2329 - _globals['_FUNCTIONREQUEST']._serialized_start=2331 - _globals['_FUNCTIONREQUEST']._serialized_end=2409 - _globals['_FUNCTIONRESPONSE']._serialized_start=2411 - _globals['_FUNCTIONRESPONSE']._serialized_end=2488 - _globals['_MODELSERVICE']._serialized_start=2957 - _globals['_MODELSERVICE']._serialized_end=3079 - _globals['_CONTROL']._serialized_start=3082 - _globals['_CONTROL']._serialized_end=3398 - _globals['_REDUCER']._serialized_start=3400 - _globals['_REDUCER']._serialized_end=3486 - _globals['_CONNECTOR']._serialized_start=3489 - _globals['_CONNECTOR']._serialized_end=3916 - _globals['_COMBINER']._serialized_start=3919 - _globals['_COMBINER']._serialized_end=4110 - _globals['_FUNCTIONSERVICE']._serialized_start=4112 - _globals['_FUNCTIONSERVICE']._serialized_end=4195 + _globals['_PROVIDEDFUNCTIONSREQUEST']._serialized_start=2331 + _globals['_PROVIDEDFUNCTIONSREQUEST']._serialized_end=2380 + _globals['_PROVIDEDFUNCTIONSRESPONSE']._serialized_start=2383 + _globals['_PROVIDEDFUNCTIONSRESPONSE']._serialized_end=2555 + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_start=2498 + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_end=2555 + _globals['_CLIENTCONFIGREQUEST']._serialized_start=2557 + _globals['_CLIENTCONFIGREQUEST']._serialized_end=2592 + _globals['_CLIENTCONFIGRESPONSE']._serialized_start=2594 + _globals['_CLIENTCONFIGRESPONSE']._serialized_end=2639 + _globals['_CLIENTSELECTIONREQUEST']._serialized_start=2641 + _globals['_CLIENTSELECTIONREQUEST']._serialized_end=2685 + _globals['_CLIENTSELECTIONRESPONSE']._serialized_start=2687 + _globals['_CLIENTSELECTIONRESPONSE']._serialized_end=2732 + _globals['_CLIENTMETAREQUEST']._serialized_start=2734 + _globals['_CLIENTMETAREQUEST']._serialized_end=2790 + _globals['_CLIENTMETARESPONSE']._serialized_start=2792 + _globals['_CLIENTMETARESPONSE']._serialized_end=2828 + _globals['_AGGREGATIONREQUEST']._serialized_start=2830 + _globals['_AGGREGATIONREQUEST']._serialized_end=2902 + _globals['_AGGREGATIONRESPONSE']._serialized_start=2904 + _globals['_AGGREGATIONRESPONSE']._serialized_end=2939 + _globals['_MODELSERVICE']._serialized_start=3408 + _globals['_MODELSERVICE']._serialized_end=3530 + _globals['_CONTROL']._serialized_start=3533 + _globals['_CONTROL']._serialized_end=3848 + _globals['_REDUCER']._serialized_start=3850 + _globals['_REDUCER']._serialized_end=3936 + _globals['_CONNECTOR']._serialized_start=3939 + _globals['_CONNECTOR']._serialized_end=4366 + _globals['_COMBINER']._serialized_start=4369 + _globals['_COMBINER']._serialized_end=4560 + _globals['_FUNCTIONSERVICE']._serialized_start=4563 + _globals['_FUNCTIONSERVICE']._serialized_end=4984 # @@protoc_insertion_point(module_scope) diff --git a/fedn/network/grpc/fedn_pb2_grpc.py b/fedn/network/grpc/fedn_pb2_grpc.py index 63bf1f625..7f6998bbf 100644 --- a/fedn/network/grpc/fedn_pb2_grpc.py +++ b/fedn/network/grpc/fedn_pb2_grpc.py @@ -128,8 +128,8 @@ def __init__(self, channel): request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, ) - self.SetFunctionProvider = channel.unary_unary( - '/fedn.Control/SetFunctionProvider', + self.SetServerFunctions = channel.unary_unary( + '/fedn.Control/SetServerFunctions', request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, ) @@ -162,7 +162,7 @@ def SetAggregator(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def SetFunctionProvider(self, request, context): + def SetServerFunctions(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -191,8 +191,8 @@ def add_ControlServicer_to_server(servicer, server): request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, ), - 'SetFunctionProvider': grpc.unary_unary_rpc_method_handler( - servicer.SetFunctionProvider, + 'SetServerFunctions': grpc.unary_unary_rpc_method_handler( + servicer.SetServerFunctions, request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.FromString, response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.SerializeToString, ), @@ -275,7 +275,7 @@ def SetAggregator(request, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def SetFunctionProvider(request, + def SetServerFunctions(request, target, options=(), channel_credentials=None, @@ -285,7 +285,7 @@ def SetFunctionProvider(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/fedn.Control/SetFunctionProvider', + return grpc.experimental.unary_unary(request, target, '/fedn.Control/SetServerFunctions', fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlRequest.SerializeToString, fedn_dot_network_dot_grpc_dot_fedn__pb2.ControlResponse.FromString, options, channel_credentials, @@ -754,17 +754,61 @@ def __init__(self, channel): Args: channel: A grpc.Channel. """ - self.ExecuteFunction = channel.unary_unary( - '/fedn.FunctionService/ExecuteFunction', - request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionRequest.SerializeToString, - response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionResponse.FromString, + self.HandleProvidedFunctions = channel.unary_unary( + '/fedn.FunctionService/HandleProvidedFunctions', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ProvidedFunctionsRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ProvidedFunctionsResponse.FromString, + ) + self.HandleClientConfig = channel.stream_unary( + '/fedn.FunctionService/HandleClientConfig', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientConfigRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientConfigResponse.FromString, + ) + self.HandleClientSelection = channel.unary_unary( + '/fedn.FunctionService/HandleClientSelection', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientSelectionRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientSelectionResponse.FromString, + ) + self.HandleMetadata = channel.unary_unary( + '/fedn.FunctionService/HandleMetadata', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaResponse.FromString, + ) + self.HandleAggregation = channel.stream_stream( + '/fedn.FunctionService/HandleAggregation', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationResponse.FromString, ) class FunctionServiceServicer(object): """Missing associated documentation comment in .proto file.""" - def ExecuteFunction(self, request, context): + def HandleProvidedFunctions(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def HandleClientConfig(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def HandleClientSelection(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def HandleMetadata(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def HandleAggregation(self, request_iterator, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -773,10 +817,30 @@ def ExecuteFunction(self, request, context): def add_FunctionServiceServicer_to_server(servicer, server): rpc_method_handlers = { - 'ExecuteFunction': grpc.unary_unary_rpc_method_handler( - servicer.ExecuteFunction, - request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionRequest.FromString, - response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionResponse.SerializeToString, + 'HandleProvidedFunctions': grpc.unary_unary_rpc_method_handler( + servicer.HandleProvidedFunctions, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ProvidedFunctionsRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ProvidedFunctionsResponse.SerializeToString, + ), + 'HandleClientConfig': grpc.stream_unary_rpc_method_handler( + servicer.HandleClientConfig, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientConfigRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientConfigResponse.SerializeToString, + ), + 'HandleClientSelection': grpc.unary_unary_rpc_method_handler( + servicer.HandleClientSelection, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientSelectionRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientSelectionResponse.SerializeToString, + ), + 'HandleMetadata': grpc.unary_unary_rpc_method_handler( + servicer.HandleMetadata, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaResponse.SerializeToString, + ), + 'HandleAggregation': grpc.stream_stream_rpc_method_handler( + servicer.HandleAggregation, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -789,7 +853,75 @@ class FunctionService(object): """Missing associated documentation comment in .proto file.""" @staticmethod - def ExecuteFunction(request, + def HandleProvidedFunctions(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/fedn.FunctionService/HandleProvidedFunctions', + fedn_dot_network_dot_grpc_dot_fedn__pb2.ProvidedFunctionsRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ProvidedFunctionsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def HandleClientConfig(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_unary(request_iterator, target, '/fedn.FunctionService/HandleClientConfig', + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientConfigRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientConfigResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def HandleClientSelection(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/fedn.FunctionService/HandleClientSelection', + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientSelectionRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientSelectionResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def HandleMetadata(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/fedn.FunctionService/HandleMetadata', + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def HandleAggregation(request_iterator, target, options=(), channel_credentials=None, @@ -799,8 +931,8 @@ def ExecuteFunction(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/fedn.FunctionService/ExecuteFunction', - fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionRequest.SerializeToString, - fedn_dot_network_dot_grpc_dot_fedn__pb2.FunctionResponse.FromString, + return grpc.experimental.stream_stream(request_iterator, target, '/fedn.FunctionService/HandleAggregation', + fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From ed468cf17bcb9db4be5f9f13e2816f4b6016f1d3 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 09:53:22 +0200 Subject: [PATCH 09/21] fix test --- fedn/network/combiner/aggregators/aggregatorbase.py | 1 - fedn/network/combiner/hooks/hook_client.py | 1 - fedn/network/combiner/hooks/hooks.py | 1 - fedn/network/combiner/hooks/serverfunctionsbase.py | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/fedn/network/combiner/aggregators/aggregatorbase.py b/fedn/network/combiner/aggregators/aggregatorbase.py index 19006d8a8..6866c6260 100644 --- a/fedn/network/combiner/aggregators/aggregatorbase.py +++ b/fedn/network/combiner/aggregators/aggregatorbase.py @@ -1,5 +1,4 @@ import importlib -import queue from abc import ABC, abstractmethod from fedn.network.combiner.updatehandler import UpdateHandler diff --git a/fedn/network/combiner/hooks/hook_client.py b/fedn/network/combiner/hooks/hook_client.py index 77965981e..1e50eb243 100644 --- a/fedn/network/combiner/hooks/hook_client.py +++ b/fedn/network/combiner/hooks/hook_client.py @@ -1,6 +1,5 @@ import json import os -from io import BytesIO import grpc diff --git a/fedn/network/combiner/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py index 237a0f2e0..9fa8cc741 100644 --- a/fedn/network/combiner/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -1,4 +1,3 @@ -import inspect import json from concurrent import futures from io import BytesIO diff --git a/fedn/network/combiner/hooks/serverfunctionsbase.py b/fedn/network/combiner/hooks/serverfunctionsbase.py index b0e4efb67..83ef9c03e 100644 --- a/fedn/network/combiner/hooks/serverfunctionsbase.py +++ b/fedn/network/combiner/hooks/serverfunctionsbase.py @@ -18,7 +18,7 @@ def __init__(self) -> None: """ pass - def aggregate(self, previous_global: list[np.ndarray], client_updates: Dict[str, Tuple[List[np.ndarray], dict]]) -> list[np.ndarray]: + def aggregate(self, previous_global: List[np.ndarray], client_updates: Dict[str, Tuple[List[np.ndarray], Dict]]) -> List[np.ndarray]: """Aggregates a list of parameters from clients. Args: From 3b5a1447526da21738afad146b073925d501e75d Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 09:57:05 +0200 Subject: [PATCH 10/21] fix test --- examples/server-functions/server_functions.py | 4 ++-- fedn/network/combiner/hooks/serverfunctionsbase.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/server-functions/server_functions.py b/examples/server-functions/server_functions.py index 65a75db83..e1a1e5f8d 100644 --- a/examples/server-functions/server_functions.py +++ b/examples/server-functions/server_functions.py @@ -14,13 +14,13 @@ def __init__(self) -> None: # Skip any function to use the default FEDn implementation for the function. # Called first in the beggining of a round to select clients. - def client_selection(self, client_ids: list[str]) -> list: + def client_selection(self, client_ids: List[str]) -> List: # Pick 10 random clients client_ids = random.sample(client_ids, min(len(client_ids), 10)) # noqa: F405 return client_ids # Called secondly before sending the global model. - def client_config(self, global_model: list[np.ndarray]) -> dict: + def client_config(self, global_model: List[np.ndarray]) -> dict: # Decrease learning rate every 10 rounds if self.round % 10 == 0: self.lr = self.lr * 0.1 diff --git a/fedn/network/combiner/hooks/serverfunctionsbase.py b/fedn/network/combiner/hooks/serverfunctionsbase.py index 83ef9c03e..b576ec75b 100644 --- a/fedn/network/combiner/hooks/serverfunctionsbase.py +++ b/fedn/network/combiner/hooks/serverfunctionsbase.py @@ -38,7 +38,7 @@ def aggregate(self, previous_global: List[np.ndarray], client_updates: Dict[str, """ pass - def client_config(self, global_model: list[np.ndarray]) -> dict: + def client_config(self, global_model: List[np.ndarray]) -> Dict: """Returns metadata related to the model, which gets distributed to the clients. The dictionary may only contain primitive types. @@ -54,7 +54,7 @@ def client_config(self, global_model: list[np.ndarray]) -> dict: """ pass - def client_selection(self, client_ids: list[str]) -> list: + def client_selection(self, client_ids: List[str]) -> List: """Returns a list of client_id's of which clients to be used for the next training request. Args: From cfac8a90e0e634b0b9f8be78b5d9c794c661854a Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 10:25:00 +0200 Subject: [PATCH 11/21] cleanup --- fedn/network/clients/client.py | 2 +- fedn/network/combiner/hooks/hooks.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index 953f724e6..ae23b59af 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -623,7 +623,7 @@ def process_request(self): if task_type == "train": tic = time.time() self.state = ClientState.training - client_config = json.loads(request.data)["client_config"] + client_config = json.loads(request.data).get("client_config", {}) model_id, meta = self._process_training_request(request.model_id, session_id=request.session_id, client_config=client_config) if meta is not None: diff --git a/fedn/network/combiner/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py index 9fa8cc741..5c749a63a 100644 --- a/fedn/network/combiner/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -84,7 +84,6 @@ def HandleProvidedFunctions(self, request, context): # check if aggregation is available try: ret = self.server_functions.aggregate(0, 0) - logger.info(f"ret : {ret}") if ret is None: implemented_functions["aggregate"] = False else: From 68160c4aaec653a87638828eb160aa8c8c2d95a6 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 11:40:19 +0200 Subject: [PATCH 12/21] cleanup --- fedn/network/combiner/aggregators/fedopt.py | 2 +- fedn/network/combiner/hooks/hook_client.py | 27 ++++++-- fedn/network/combiner/hooks/hooks.py | 67 +++++++++++++++++-- .../combiner/hooks/serverfunctionsbase.py | 3 - 4 files changed, 83 insertions(+), 16 deletions(-) diff --git a/fedn/network/combiner/aggregators/fedopt.py b/fedn/network/combiner/aggregators/fedopt.py index 9e57ed9ca..1558b0b03 100644 --- a/fedn/network/combiner/aggregators/fedopt.py +++ b/fedn/network/combiner/aggregators/fedopt.py @@ -109,7 +109,7 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): total_examples += metadata["num_examples"] if nr_aggregated_models == 0: - model_old = deepcopy(model_next) + model_old = self.update_handler.load_model(helper, model_update.model_id) pseudo_gradient = helper.subtract(model_next, model_old) else: pseudo_gradient_next = helper.subtract(model_next, model_old) diff --git a/fedn/network/combiner/hooks/hook_client.py b/fedn/network/combiner/hooks/hook_client.py index 1e50eb243..f1875d265 100644 --- a/fedn/network/combiner/hooks/hook_client.py +++ b/fedn/network/combiner/hooks/hook_client.py @@ -13,21 +13,35 @@ class CombinerHookInterface: + """Combiner to server function hooks client.""" + def __init__(self): - logger.info("Starting hook client") + """Initialize CombinerHookInterface client.""" self.hook_service_host = os.getenv("HOOK_SERVICE_HOST", "hook:12081") self.channel = grpc.insecure_channel(self.hook_service_host) self.stub = rpc.FunctionServiceStub(self.channel) def provided_functions(self, server_functions: str): - """Communicates to hook container and asks which functions are available.""" + """Communicates to hook container and asks which functions are available. + + :param server_functions: String version of an implementation of the ServerFunctionsBase interface. + :type server_functions: :str: + :return: dictionary specifing which functions are implemented. + :rtype: dict + """ request = fedn.ProvidedFunctionsRequest(function_code=server_functions) response = self.stub.HandleProvidedFunctions(request) return response.available_functions def client_config(self, global_model) -> dict: - """Communicates to hook container to get a client config.""" + """Communicates to hook container to get a client config. + + :param global_model: The global model that will be distributed to clients. + :type global_model: :bytes: + :return: config that will be distributed to clients. + :rtype: dict + """ request_function = fedn.ClientConfigRequest args = {} model = model_as_bytesIO(global_model) @@ -40,9 +54,12 @@ def client_selection(self, clients: list) -> list: return json.loads(response.client_ids) def aggregate(self, previous_global, update_handler: UpdateHandler, helper, delete_models: bool): - """Aggregation call to the hook functions. + """Aggregation call to the hook functions. Sends models in chunks, then asks for aggregation. - Sends models in chunks, then asks for aggregation. + :param global_model: The global model that will be distributed to clients. + :type global_model: :bytes: + :return: config that will be distributed to clients. + :rtype: dict """ data = {} data["time_model_load"] = 0.0 diff --git a/fedn/network/combiner/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py index 5c749a63a..1c9ca9bcb 100644 --- a/fedn/network/combiner/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -18,34 +18,77 @@ class FunctionServiceServicer(rpc.FunctionServiceServicer): + """Function service running in an environment combined with each combiner. + + Receiving requests from the combiner. + """ + def __init__(self) -> None: + """Initialize long-running Function server.""" super().__init__() self.helper = Helper() self.server_functions: ServerFunctionsBase = None + self.server_functions_code: str = None self.client_updates = {} - def HandleClientConfig(self, request_iterator, context): + def HandleClientConfig(self, request_iterator: fedn.ClientConfigRequest, context): + """Distribute client configs to clients from user defined code. + + :param request_iterator: the client config request + :type request_iterator: :class:`fedn.network.grpc.fedn_pb2.ClientConfigRequest` + :param context: the context (unused) + :type context: :class:`grpc._server._Context` + :return: the client config response + :rtype: :class:`fedn.network.grpc.fedn_pb2.ClientConfigResponse` + """ logger.info("Received client config request.") model = self.unpack_model(request_iterator) client_config = self.server_functions.client_config(global_model=model) logger.info(f"Client config response: {client_config}") return fedn.ClientConfigResponse(client_config=json.dumps(client_config)) - def HandleClientSelection(self, request, context): + def HandleClientSelection(self, request: fedn.ClientSelectionRequest, context): + """Handle client selection from user defined code. + + :param request: the client selection request + :type request: :class:`fedn.network.grpc.fedn_pb2.fedn.ClientSelectionRequest` + :param context: the context (unused) + :type context: :class:`grpc._server._Context` + :return: the client selection response + :rtype: :class:`fedn.network.grpc.fedn_pb2.ClientSelectionResponse` + """ logger.info("Received client selection request.") client_ids = json.loads(request.client_ids) client_ids = self.server_functions.client_selection(client_ids) logger.info(f"Clients selected: {client_ids}") return fedn.ClientSelectionResponse(client_ids=json.dumps(client_ids)) - def HandleMetadata(self, request, context): + def HandleMetadata(self, request: fedn.ClientMetaRequest, context): + """Store client metadata from a request. + + :param request: the client meta request + :type request: :class:`fedn.network.grpc.fedn_pb2.fedn.ClientMetaRequest` + :param context: the context (unused) + :type context: :class:`grpc._server._Context` + :return: the client meta response + :rtype: :class:`fedn.network.grpc.fedn_pb2.ClientMetaResponse` + """ client_id = request.client_id metadata = json.loads(request.metadata) self.client_updates[client_id] = self.client_updates.get(client_id, []) + [metadata] return fedn.ClientMetaResponse(status="Metadata stored") - def HandleAggregation(self, request_iterator, context): + def HandleAggregation(self, request_iterator: fedn.AggregationRequest, context): + """Receive and store models and aggregate based on user-defined code when specified in the request. + + :param request_iterator: the aggregation request + :type request_iterator: :class:`fedn.network.grpc.fedn_pb2.fedn.AggregationRequest` + :param context: the context (unused) + :type context: :class:`grpc._server._Context` + :return: the aggregation response (aggregated model or None) + :rtype: :class:`fedn.network.grpc.fedn_pb2.AggregationResponse` + """ # check what type of request for request in request_iterator: if request.aggregate: @@ -66,11 +109,20 @@ def HandleAggregation(self, request_iterator, context): self.client_updates[client_id] = [model] + self.client_updates.get(client_id, []) return fedn.AggregationResponse(data=None) - def HandleProvidedFunctions(self, request, context): - """Handles the 'provided_functions' request. Sends back which functions are available.""" + def HandleProvidedFunctions(self, request: fedn.ProvidedFunctionsResponse, context): + """Handles the 'provided_functions' request. Sends back which functions are available. + + :param request: the provided function request + :type request: :class:`fedn.network.grpc.fedn_pb2.fedn.ProvidedFunctionsRequest` + :param context: the context (unused) + :type context: :class:`grpc._server._Context` + :return: dict with str -> bool for which functions are available + :rtype: :class:`fedn.network.grpc.fedn_pb2.ProvidedFunctionsResponse` + """ logger.info("Receieved provided functions request.") server_functions_code = request.function_code - if self.server_functions is None: + if self.server_functions is None or server_functions_code != self.server_functions_code: + self.server_functions_code = server_functions_code # this will create a new user defined instance of the ServerFunctions class. try: namespace = {} @@ -137,6 +189,7 @@ def unpack_model(self, request_iterator): def serve(): + """Start the hooks service.""" server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) rpc.add_FunctionServiceServicer_to_server(FunctionServiceServicer(), server) server.add_insecure_port("[::]:12081") diff --git a/fedn/network/combiner/hooks/serverfunctionsbase.py b/fedn/network/combiner/hooks/serverfunctionsbase.py index b576ec75b..c276941d7 100644 --- a/fedn/network/combiner/hooks/serverfunctionsbase.py +++ b/fedn/network/combiner/hooks/serverfunctionsbase.py @@ -7,9 +7,6 @@ class ServerFunctionsBase(ABC): """Base class that defines the structure for the Server Functions. Override these functions to add to the server workflow. - - Available packages are: - numpy as np, and most common python built in packages. """ def __init__(self) -> None: From 3d82e45e683da2fe66c6db96dd14126d7bf7caa3 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 12:20:04 +0200 Subject: [PATCH 13/21] fix linting --- fedn/network/combiner/aggregators/fedopt.py | 1 - fedn/network/combiner/hooks/hook_client.py | 1 - 2 files changed, 2 deletions(-) diff --git a/fedn/network/combiner/aggregators/fedopt.py b/fedn/network/combiner/aggregators/fedopt.py index 1558b0b03..ac53795e7 100644 --- a/fedn/network/combiner/aggregators/fedopt.py +++ b/fedn/network/combiner/aggregators/fedopt.py @@ -1,5 +1,4 @@ import math -from copy import deepcopy from fedn.common.exceptions import InvalidParameterError from fedn.common.log_config import logger diff --git a/fedn/network/combiner/hooks/hook_client.py b/fedn/network/combiner/hooks/hook_client.py index f1875d265..9fedd27b0 100644 --- a/fedn/network/combiner/hooks/hook_client.py +++ b/fedn/network/combiner/hooks/hook_client.py @@ -5,7 +5,6 @@ import fedn.network.grpc.fedn_pb2 as fedn import fedn.network.grpc.fedn_pb2_grpc as rpc -from fedn.common.log_config import logger from fedn.network.combiner.modelservice import bytesIO_request_generator, load_model_from_bytes, model_as_bytesIO from fedn.network.combiner.updatehandler import UpdateHandler From 8e587e4343dcc6447bbe52a5dfa81e981c12de25 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 12:22:52 +0200 Subject: [PATCH 14/21] add server-functions to test --- .github/workflows/integration-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index ed354a78b..c352bb27c 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -23,6 +23,7 @@ jobs: to_test: - "mnist-keras numpyhelper" - "mnist-pytorch numpyhelper" + - "server-functions numpyhelper" python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: - ubuntu-22.04 From 0e3a487e67f7b9074b3363e83b0192b03b5c2a28 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 8 Oct 2024 13:22:46 +0200 Subject: [PATCH 15/21] cleanup --- .github/workflows/integration-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index c352bb27c..ed354a78b 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -23,7 +23,6 @@ jobs: to_test: - "mnist-keras numpyhelper" - "mnist-pytorch numpyhelper" - - "server-functions numpyhelper" python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: - ubuntu-22.04 From 083a92c091db643c5c7e1fd39b1525d0b354494e Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 15 Oct 2024 06:16:39 +0200 Subject: [PATCH 16/21] update README --- examples/server-functions/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server-functions/README.rst b/examples/server-functions/README.rst index 43c9c2fb4..c594fac28 100644 --- a/examples/server-functions/README.rst +++ b/examples/server-functions/README.rst @@ -8,4 +8,4 @@ README Will be updated after studio update. To run with server functions: from server_functions import ServerFunctions -client.start_session(aggregator="custom", server_functions=ServerFunctions) +client.start_session(server_functions=ServerFunctions) \ No newline at end of file From 49b5b9fd8704b3c96732319a69bfcc86980ef85b Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Wed, 16 Oct 2024 10:55:40 +0200 Subject: [PATCH 17/21] bug fix and code improvements --- examples/server-functions/server_functions.py | 22 +++--- fedn/network/combiner/hooks/hook_client.py | 33 +++++---- fedn/network/combiner/hooks/hooks.py | 71 +++++++------------ fedn/network/combiner/modelservice.py | 35 ++++++++- fedn/network/combiner/roundhandler.py | 1 + fedn/network/combiner/updatehandler.py | 14 ++-- fedn/network/grpc/fedn.proto | 16 +++-- fedn/network/grpc/fedn_pb2.py | 62 ++++++++-------- fedn/network/grpc/fedn_pb2_grpc.py | 43 +++++++++-- 9 files changed, 177 insertions(+), 120 deletions(-) diff --git a/examples/server-functions/server_functions.py b/examples/server-functions/server_functions.py index e1a1e5f8d..f760bec93 100644 --- a/examples/server-functions/server_functions.py +++ b/examples/server-functions/server_functions.py @@ -9,7 +9,7 @@ class ServerFunctions(ServerFunctionsBase): def __init__(self) -> None: # You can keep a state between different functions to have them work together. self.round = 0 - self.lr = 0.01 + self.lr = 0.1 # Skip any function to use the default FEDn implementation for the function. @@ -33,15 +33,15 @@ def aggregate(self, previous_global: List[np.ndarray], client_updates: Dict[str, # Weighted fedavg implementation. weighted_sum = [np.zeros_like(param) for param in previous_global] total_weight = 0 + logger.info("updateees ye") for client_id, (client_parameters, metadata) in client_updates.items(): + logger.info(f"metadata: {metadata}") num_examples = metadata.get("num_examples", 0) - if num_examples > 0: - total_weight += num_examples - for i in range(len(weighted_sum)): - weighted_sum[i] += client_parameters[i] * num_examples - if total_weight > 0: - averaged_updates = [weighted / total_weight for weighted in weighted_sum] - return averaged_updates - # Need to use logger instead of prints. - logger.info("Aggregation failed, using previous global model.") - return previous_global + logger.info(f"num_examples: {num_examples}") + total_weight += num_examples + for i in range(len(weighted_sum)): + weighted_sum[i] += client_parameters[i] * num_examples + + logger.info("Models aggregated") + averaged_updates = [weighted / total_weight for weighted in weighted_sum] + return averaged_updates diff --git a/fedn/network/combiner/hooks/hook_client.py b/fedn/network/combiner/hooks/hook_client.py index 9fedd27b0..8b8a9b6bb 100644 --- a/fedn/network/combiner/hooks/hook_client.py +++ b/fedn/network/combiner/hooks/hook_client.py @@ -5,7 +5,8 @@ import fedn.network.grpc.fedn_pb2 as fedn import fedn.network.grpc.fedn_pb2_grpc as rpc -from fedn.network.combiner.modelservice import bytesIO_request_generator, load_model_from_bytes, model_as_bytesIO +from fedn.common.log_config import logger +from fedn.network.combiner.modelservice import bytesIO_request_generator, model_as_bytesIO, unpack_model from fedn.network.combiner.updatehandler import UpdateHandler CHUNK_SIZE = 1024 * 1024 @@ -64,28 +65,30 @@ def aggregate(self, previous_global, update_handler: UpdateHandler, helper, dele data["time_model_load"] = 0.0 data["time_model_aggregation"] = 0.0 # send previous global - request_function = fedn.AggregationRequest - args = {"id": "global_model", "aggregate": False} - model = model_as_bytesIO(previous_global) - self.stub.HandleAggregation(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) + request_function = fedn.StoreModelRequest + args = {"id": "global_model"} + response = self.stub.HandleStoreModel(bytesIO_request_generator(mdl=previous_global, request_function=request_function, args=args)) + logger.info(f"Store model response: {response.status}") # send client models and metadata updates = update_handler.get_model_updates() for update in updates: - model, metadata = update_handler.load_model_update(update, helper) + metadata = json.loads(update.meta)["training_metadata"] + model = update_handler.load_model_update_bytesIO(update.model_update_id) # send metadata client_id = update.sender.client_id - request = fedn.ClientMetaRequest(metadata=metadata, client_id=client_id) - self.stub.HandleMetadata(request) + request = fedn.ClientMetaRequest(metadata=json.dumps(metadata), client_id=client_id) + response = self.stub.HandleMetadata(request) # send client model - model = model_as_bytesIO(model) - args = {"id": client_id, "aggregate": False} - request_function = fedn.AggregationRequest - self.stub.HandleAggregation(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) + args = {"id": client_id} + request_function = fedn.StoreModelRequest + response = self.stub.HandleStoreModel(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) + logger.info(f"Store model response: {response.status}") if delete_models: # delete model from disk update_handler.delete_model(model_update=update) # ask for aggregation - request = fedn.AggregationRequest(data=None, client_id="", aggregate=True) - response = self.stub.HandleAggregation(request) + request = fedn.AggregationRequest(aggregate="aggregate") + response_generator = self.stub.HandleAggregation(request) data["nr_aggregated_models"] = len(updates) - return load_model_from_bytes(response.data, helper), data + model, _ = unpack_model(response_generator, helper) + return model, data diff --git a/fedn/network/combiner/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py index 1c9ca9bcb..666e27cc4 100644 --- a/fedn/network/combiner/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -1,6 +1,5 @@ import json from concurrent import futures -from io import BytesIO import grpc @@ -10,7 +9,7 @@ # imports for user code from fedn.network.combiner.hooks.allowed_import import Dict, List, ServerFunctionsBase, Tuple, np, random # noqa: F401 -from fedn.network.combiner.modelservice import bytesIO_request_generator, load_model_from_bytes, model_as_bytesIO +from fedn.network.combiner.modelservice import bytesIO_request_generator, model_as_bytesIO, unpack_model from fedn.utils.helpers.plugins.numpyhelper import Helper CHUNK_SIZE = 1024 * 1024 @@ -43,7 +42,7 @@ def HandleClientConfig(self, request_iterator: fedn.ClientConfigRequest, context :rtype: :class:`fedn.network.grpc.fedn_pb2.ClientConfigResponse` """ logger.info("Received client config request.") - model = self.unpack_model(request_iterator) + model, _ = unpack_model(request_iterator, self.helper) client_config = self.server_functions.client_config(global_model=model) logger.info(f"Client config response: {client_config}") return fedn.ClientConfigResponse(client_config=json.dumps(client_config)) @@ -74,12 +73,24 @@ def HandleMetadata(self, request: fedn.ClientMetaRequest, context): :return: the client meta response :rtype: :class:`fedn.network.grpc.fedn_pb2.ClientMetaResponse` """ + logger.info("Received metadata") client_id = request.client_id metadata = json.loads(request.metadata) self.client_updates[client_id] = self.client_updates.get(client_id, []) + [metadata] return fedn.ClientMetaResponse(status="Metadata stored") - def HandleAggregation(self, request_iterator: fedn.AggregationRequest, context): + def HandleStoreModel(self, request_iterator, context): + model, final_request = unpack_model(request_iterator, self.helper) + client_id = final_request.id + if client_id == "global_model": + logger.info("Received previous global model") + self.previous_global = model + else: + logger.info("Received client model") + self.client_updates[client_id] = [model] + self.client_updates.get(client_id, []) + return fedn.StoreModelResponse(status=f"Received model originating from {client_id}") + + def HandleAggregation(self, request, context): """Receive and store models and aggregate based on user-defined code when specified in the request. :param request_iterator: the aggregation request @@ -89,25 +100,15 @@ def HandleAggregation(self, request_iterator: fedn.AggregationRequest, context): :return: the aggregation response (aggregated model or None) :rtype: :class:`fedn.network.grpc.fedn_pb2.AggregationResponse` """ - # check what type of request - for request in request_iterator: - if request.aggregate: - logger.info("Received aggregation request.") - aggregated_model = self.server_functions.aggregate(self.previous_global, self.client_updates) - aggregated_model = model_as_bytesIO(aggregated_model) - request_function = fedn.AggregationResponse - logger.info("Returning aggregate model.") - return bytesIO_request_generator(mdl=aggregated_model, request_function=request_function, args={}) - client_id = request.client_id - break - logger.info(f"Received request to store model originating from: {client_id}") - model = self.unpack_model(request_iterator) - - if client_id == "global_model": - self.previous_global = model - else: - self.client_updates[client_id] = [model] + self.client_updates.get(client_id, []) - return fedn.AggregationResponse(data=None) + logger.info(f"Receieved aggregation request: {request.aggregate}") + aggregated_model = self.server_functions.aggregate(self.previous_global, self.client_updates) + model_bytesIO = model_as_bytesIO(aggregated_model, self.helper) + request_function = fedn.AggregationResponse + self.client_updates = {} + logger.info("Returning aggregate model.") + response_generator = bytesIO_request_generator(mdl=model_bytesIO, request_function=request_function, args={}) + for response in response_generator: + yield response def HandleProvidedFunctions(self, request: fedn.ProvidedFunctionsResponse, context): """Handles the 'provided_functions' request. Sends back which functions are available. @@ -163,30 +164,6 @@ def HandleProvidedFunctions(self, request: fedn.ProvidedFunctionsResponse, conte logger.info(f"Provided function: {implemented_functions}") return fedn.ProvidedFunctionsResponse(available_functions=implemented_functions) - def unpack_model(self, request_iterator): - """Unpack the incoming model sent in chunks from the request iterator. - - :param request_iterator: A streaming iterator from the gRPC service. - :return: The reconstructed model parameters. - """ - model_buffer = BytesIO() - try: - for request in request_iterator: - if request.data: - model_buffer.write(request.data) - except MemoryError as e: - print(f"Memory error occured when loading model, reach out to the FEDn team if you need a solution to this. {e}") - raise - except Exception as e: - print(f"Exception occured during model loading: {e}") - raise - - model_buffer.seek(0) - - model_bytes = model_buffer.getvalue() - - return load_model_from_bytes(model_bytes, self.helper) - def serve(): """Start the hooks service.""" diff --git a/fedn/network/combiner/modelservice.py b/fedn/network/combiner/modelservice.py index 7571182af..89901dabb 100644 --- a/fedn/network/combiner/modelservice.py +++ b/fedn/network/combiner/modelservice.py @@ -2,6 +2,8 @@ import tempfile from io import BytesIO +import numpy as np + import fedn.network.grpc.fedn_pb2 as fedn import fedn.network.grpc.fedn_pb2_grpc as rpc from fedn.common.log_config import logger @@ -52,9 +54,13 @@ def bytesIO_request_generator(mdl, request_function, args): break -def model_as_bytesIO(model): +def model_as_bytesIO(model, helper=None): if isinstance(model, list): - serialize_model_to_BytesIO(model) + bt = BytesIO() + model_dict = {str(i): w for i, w in enumerate(model)} + np.savez_compressed(bt, **model_dict) + bt.seek(0) + return bt if not isinstance(model, BytesIO): bt = BytesIO() @@ -69,6 +75,31 @@ def model_as_bytesIO(model): return bt +def unpack_model(request_iterator, helper): + """Unpack an incoming model sent in chunks from a request iterator. + + :param request_iterator: A streaming iterator from an gRPC service. + :return: The reconstructed model parameters. + """ + model_buffer = BytesIO() + try: + for request in request_iterator: + if request.data: + model_buffer.write(request.data) + except MemoryError as e: + logger.error(f"Memory error occured when loading model, reach out to the FEDn team if you need a solution to this. {e}") + raise + except Exception as e: + logger.error(f"Exception occured during model loading: {e}") + raise + + model_buffer.seek(0) + + model_bytes = model_buffer.getvalue() + + return load_model_from_bytes(model_bytes, helper), request + + def get_tmp_path(): """Return a temporary output path compatible with save_model, load_model.""" fd, path = tempfile.mkstemp() diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index a877f9efb..0d441bda0 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -175,6 +175,7 @@ def _training_round(self, config, clients, provided_functions): model, data = self.aggregator.combine_models(helper=helper, delete_models=delete_models, parameters=parameters) except Exception as e: logger.warning("AGGREGATION FAILED AT COMBINER! {}".format(e)) + raise self.update_handler.flush() diff --git a/fedn/network/combiner/updatehandler.py b/fedn/network/combiner/updatehandler.py index d2f395688..08e5696a1 100644 --- a/fedn/network/combiner/updatehandler.py +++ b/fedn/network/combiner/updatehandler.py @@ -72,7 +72,7 @@ def _validate_model_update(self, model_update): _ = data["num_examples"] except KeyError: tb = traceback.format_exc() - logger.error("AGGREGATOR({}): Invalid model update, missing metadata.".format(self.name)) + logger.error("UPDATE HANDLER: Invalid model update, missing metadata.") logger.error(tb) return False return True @@ -117,7 +117,7 @@ def load_model_update_byte(self, model_update): :rtype: tuple """ model_id = model_update.model_update_id - model = self._load_model_update_str(model_id).getbuffer() + model = self.load_model_update_bytesIO(model_id).getbuffer() # Get relevant metadata metadata = json.loads(model_update.meta) if "config" in metadata.keys(): @@ -139,18 +139,18 @@ def load_model(self, helper, model_id): :param model_id: The ID of the model update, UUID in str format :type model_id: str """ - model_str = self._load_model_update_str(model_id) - if model_str: + model_bytesIO = self.load_model_update_bytesIO(model_id) + if model_bytesIO: try: - model = load_model_from_bytes(model_str.getbuffer(), helper) + model = load_model_from_bytes(model_bytesIO.getbuffer(), helper) except IOError: - logger.warning("AGGREGATOR({}): Failed to load model!".format(self.name)) + logger.warning("UPDATE HANDLER: Failed to load model!") else: raise ModelUpdateError("Failed to load model.") return model - def _load_model_update_str(self, model_id, retry=3): + def load_model_update_bytesIO(self, model_id, retry=3): """Load model update object and return it as BytesIO. :param model_id: The ID of the model diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index f052858e3..d826e6f72 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -278,10 +278,17 @@ message ClientMetaResponse { string status = 1; } -message AggregationRequest { +message StoreModelRequest { bytes data = 1; - string client_id = 2; - bool aggregate = 3; + string id = 2; +} + +message StoreModelResponse { + string status = 1; +} + +message AggregationRequest { + string aggregate = 1; } message AggregationResponse { @@ -293,5 +300,6 @@ service FunctionService { rpc HandleClientConfig (stream ClientConfigRequest) returns (ClientConfigResponse); rpc HandleClientSelection (ClientSelectionRequest) returns (ClientSelectionResponse); rpc HandleMetadata (ClientMetaRequest) returns (ClientMetaResponse); - rpc HandleAggregation (stream AggregationRequest) returns (stream AggregationResponse); + rpc HandleStoreModel (stream StoreModelRequest) returns (StoreModelResponse); + rpc HandleAggregation (AggregationRequest) returns (stream AggregationResponse); } \ No newline at end of file diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index 6a2e94c60..7009fd5cd 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -15,7 +15,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"1\n\x18ProvidedFunctionsRequest\x12\x15\n\rfunction_code\x18\x01 \x01(\t\"\xac\x01\n\x19ProvidedFunctionsResponse\x12T\n\x13\x61vailable_functions\x18\x01 \x03(\x0b\x32\x37.fedn.ProvidedFunctionsResponse.AvailableFunctionsEntry\x1a\x39\n\x17\x41vailableFunctionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"#\n\x13\x43lientConfigRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"-\n\x14\x43lientConfigResponse\x12\x15\n\rclient_config\x18\x01 \x01(\t\",\n\x16\x43lientSelectionRequest\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"-\n\x17\x43lientSelectionResponse\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"8\n\x11\x43lientMetaRequest\x12\x10\n\x08metadata\x18\x01 \x01(\t\x12\x11\n\tclient_id\x18\x02 \x01(\t\"$\n\x12\x43lientMetaResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"H\n\x12\x41ggregationRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x11\n\tclient_id\x18\x02 \x01(\t\x12\x11\n\taggregate\x18\x03 \x01(\x08\"#\n\x13\x41ggregationResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbb\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x41\n\x12SetServerFunctions\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2\xa5\x03\n\x0f\x46unctionService\x12Z\n\x17HandleProvidedFunctions\x12\x1e.fedn.ProvidedFunctionsRequest\x1a\x1f.fedn.ProvidedFunctionsResponse\x12M\n\x12HandleClientConfig\x12\x19.fedn.ClientConfigRequest\x1a\x1a.fedn.ClientConfigResponse(\x01\x12T\n\x15HandleClientSelection\x12\x1c.fedn.ClientSelectionRequest\x1a\x1d.fedn.ClientSelectionResponse\x12\x43\n\x0eHandleMetadata\x12\x17.fedn.ClientMetaRequest\x1a\x18.fedn.ClientMetaResponse\x12L\n\x11HandleAggregation\x12\x18.fedn.AggregationRequest\x1a\x19.fedn.AggregationResponse(\x01\x30\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"1\n\x18ProvidedFunctionsRequest\x12\x15\n\rfunction_code\x18\x01 \x01(\t\"\xac\x01\n\x19ProvidedFunctionsResponse\x12T\n\x13\x61vailable_functions\x18\x01 \x03(\x0b\x32\x37.fedn.ProvidedFunctionsResponse.AvailableFunctionsEntry\x1a\x39\n\x17\x41vailableFunctionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"#\n\x13\x43lientConfigRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"-\n\x14\x43lientConfigResponse\x12\x15\n\rclient_config\x18\x01 \x01(\t\",\n\x16\x43lientSelectionRequest\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"-\n\x17\x43lientSelectionResponse\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"8\n\x11\x43lientMetaRequest\x12\x10\n\x08metadata\x18\x01 \x01(\t\x12\x11\n\tclient_id\x18\x02 \x01(\t\"$\n\x12\x43lientMetaResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"-\n\x11StoreModelRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\"$\n\x12StoreModelResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"\'\n\x12\x41ggregationRequest\x12\x11\n\taggregate\x18\x01 \x01(\t\"#\n\x13\x41ggregationResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbb\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x41\n\x12SetServerFunctions\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response2\xec\x03\n\x0f\x46unctionService\x12Z\n\x17HandleProvidedFunctions\x12\x1e.fedn.ProvidedFunctionsRequest\x1a\x1f.fedn.ProvidedFunctionsResponse\x12M\n\x12HandleClientConfig\x12\x19.fedn.ClientConfigRequest\x1a\x1a.fedn.ClientConfigResponse(\x01\x12T\n\x15HandleClientSelection\x12\x1c.fedn.ClientSelectionRequest\x1a\x1d.fedn.ClientSelectionResponse\x12\x43\n\x0eHandleMetadata\x12\x17.fedn.ClientMetaRequest\x1a\x18.fedn.ClientMetaResponse\x12G\n\x10HandleStoreModel\x12\x17.fedn.StoreModelRequest\x1a\x18.fedn.StoreModelResponse(\x01\x12J\n\x11HandleAggregation\x12\x18.fedn.AggregationRequest\x1a\x19.fedn.AggregationResponse0\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -24,18 +24,18 @@ DESCRIPTOR._options = None _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._options = None _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_options = b'8\001' - _globals['_STATUSTYPE']._serialized_start=2942 - _globals['_STATUSTYPE']._serialized_end=3074 - _globals['_QUEUE']._serialized_start=3076 - _globals['_QUEUE']._serialized_end=3112 - _globals['_MODELSTATUS']._serialized_start=3114 - _globals['_MODELSTATUS']._serialized_end=3197 - _globals['_ROLE']._serialized_start=3199 - _globals['_ROLE']._serialized_end=3255 - _globals['_COMMAND']._serialized_start=3257 - _globals['_COMMAND']._serialized_end=3331 - _globals['_CONNECTIONSTATUS']._serialized_start=3333 - _globals['_CONNECTIONSTATUS']._serialized_end=3406 + _globals['_STATUSTYPE']._serialized_start=2994 + _globals['_STATUSTYPE']._serialized_end=3126 + _globals['_QUEUE']._serialized_start=3128 + _globals['_QUEUE']._serialized_end=3164 + _globals['_MODELSTATUS']._serialized_start=3166 + _globals['_MODELSTATUS']._serialized_end=3249 + _globals['_ROLE']._serialized_start=3251 + _globals['_ROLE']._serialized_end=3307 + _globals['_COMMAND']._serialized_start=3309 + _globals['_COMMAND']._serialized_end=3383 + _globals['_CONNECTIONSTATUS']._serialized_start=3385 + _globals['_CONNECTIONSTATUS']._serialized_end=3458 _globals['_RESPONSE']._serialized_start=71 _globals['_RESPONSE']._serialized_end=129 _globals['_STATUS']._serialized_start=132 @@ -98,20 +98,24 @@ _globals['_CLIENTMETAREQUEST']._serialized_end=2790 _globals['_CLIENTMETARESPONSE']._serialized_start=2792 _globals['_CLIENTMETARESPONSE']._serialized_end=2828 - _globals['_AGGREGATIONREQUEST']._serialized_start=2830 - _globals['_AGGREGATIONREQUEST']._serialized_end=2902 - _globals['_AGGREGATIONRESPONSE']._serialized_start=2904 - _globals['_AGGREGATIONRESPONSE']._serialized_end=2939 - _globals['_MODELSERVICE']._serialized_start=3408 - _globals['_MODELSERVICE']._serialized_end=3530 - _globals['_CONTROL']._serialized_start=3533 - _globals['_CONTROL']._serialized_end=3848 - _globals['_REDUCER']._serialized_start=3850 - _globals['_REDUCER']._serialized_end=3936 - _globals['_CONNECTOR']._serialized_start=3939 - _globals['_CONNECTOR']._serialized_end=4366 - _globals['_COMBINER']._serialized_start=4369 - _globals['_COMBINER']._serialized_end=4560 - _globals['_FUNCTIONSERVICE']._serialized_start=4563 - _globals['_FUNCTIONSERVICE']._serialized_end=4984 + _globals['_STOREMODELREQUEST']._serialized_start=2830 + _globals['_STOREMODELREQUEST']._serialized_end=2875 + _globals['_STOREMODELRESPONSE']._serialized_start=2877 + _globals['_STOREMODELRESPONSE']._serialized_end=2913 + _globals['_AGGREGATIONREQUEST']._serialized_start=2915 + _globals['_AGGREGATIONREQUEST']._serialized_end=2954 + _globals['_AGGREGATIONRESPONSE']._serialized_start=2956 + _globals['_AGGREGATIONRESPONSE']._serialized_end=2991 + _globals['_MODELSERVICE']._serialized_start=3460 + _globals['_MODELSERVICE']._serialized_end=3582 + _globals['_CONTROL']._serialized_start=3585 + _globals['_CONTROL']._serialized_end=3900 + _globals['_REDUCER']._serialized_start=3902 + _globals['_REDUCER']._serialized_end=3988 + _globals['_CONNECTOR']._serialized_start=3991 + _globals['_CONNECTOR']._serialized_end=4418 + _globals['_COMBINER']._serialized_start=4421 + _globals['_COMBINER']._serialized_end=4612 + _globals['_FUNCTIONSERVICE']._serialized_start=4615 + _globals['_FUNCTIONSERVICE']._serialized_end=5107 # @@protoc_insertion_point(module_scope) diff --git a/fedn/network/grpc/fedn_pb2_grpc.py b/fedn/network/grpc/fedn_pb2_grpc.py index 7f6998bbf..d805d7c2c 100644 --- a/fedn/network/grpc/fedn_pb2_grpc.py +++ b/fedn/network/grpc/fedn_pb2_grpc.py @@ -774,7 +774,12 @@ def __init__(self, channel): request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaRequest.SerializeToString, response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaResponse.FromString, ) - self.HandleAggregation = channel.stream_stream( + self.HandleStoreModel = channel.stream_unary( + '/fedn.FunctionService/HandleStoreModel', + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.StoreModelRequest.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.StoreModelResponse.FromString, + ) + self.HandleAggregation = channel.unary_stream( '/fedn.FunctionService/HandleAggregation', request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationRequest.SerializeToString, response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationResponse.FromString, @@ -808,7 +813,13 @@ def HandleMetadata(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def HandleAggregation(self, request_iterator, context): + def HandleStoreModel(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def HandleAggregation(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -837,7 +848,12 @@ def add_FunctionServiceServicer_to_server(servicer, server): request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaRequest.FromString, response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ClientMetaResponse.SerializeToString, ), - 'HandleAggregation': grpc.stream_stream_rpc_method_handler( + 'HandleStoreModel': grpc.stream_unary_rpc_method_handler( + servicer.HandleStoreModel, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.StoreModelRequest.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.StoreModelResponse.SerializeToString, + ), + 'HandleAggregation': grpc.unary_stream_rpc_method_handler( servicer.HandleAggregation, request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationRequest.FromString, response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationResponse.SerializeToString, @@ -921,7 +937,24 @@ def HandleMetadata(request, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def HandleAggregation(request_iterator, + def HandleStoreModel(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_unary(request_iterator, target, '/fedn.FunctionService/HandleStoreModel', + fedn_dot_network_dot_grpc_dot_fedn__pb2.StoreModelRequest.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.StoreModelResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def HandleAggregation(request, target, options=(), channel_credentials=None, @@ -931,7 +964,7 @@ def HandleAggregation(request_iterator, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.stream_stream(request_iterator, target, '/fedn.FunctionService/HandleAggregation', + return grpc.experimental.unary_stream(request, target, '/fedn.FunctionService/HandleAggregation', fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationRequest.SerializeToString, fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationResponse.FromString, options, channel_credentials, From 556c7c311fcc5fb8d84f32d3314c24e612f2113d Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Mon, 28 Oct 2024 11:12:52 +0100 Subject: [PATCH 18/21] remove logs --- examples/server-functions/server_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/server-functions/server_functions.py b/examples/server-functions/server_functions.py index f760bec93..d8cf080a8 100644 --- a/examples/server-functions/server_functions.py +++ b/examples/server-functions/server_functions.py @@ -33,11 +33,8 @@ def aggregate(self, previous_global: List[np.ndarray], client_updates: Dict[str, # Weighted fedavg implementation. weighted_sum = [np.zeros_like(param) for param in previous_global] total_weight = 0 - logger.info("updateees ye") for client_id, (client_parameters, metadata) in client_updates.items(): - logger.info(f"metadata: {metadata}") - num_examples = metadata.get("num_examples", 0) - logger.info(f"num_examples: {num_examples}") + num_examples = metadata.get("num_examples", 1) total_weight += num_examples for i in range(len(weighted_sum)): weighted_sum[i] += client_parameters[i] * num_examples From bdf5054b9ef2fa77f74b293afdbc23e75946bbf3 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Tue, 29 Oct 2024 10:17:44 +0100 Subject: [PATCH 19/21] generate grpc --- fedn/network/combiner/roundhandler.py | 1 + fedn/network/grpc/fedn_pb2.py | 170 +++++++++++++++----------- fedn/network/grpc/fedn_pb2_grpc.py | 42 +++---- 3 files changed, 122 insertions(+), 91 deletions(-) diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 752c3584b..050794fd5 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -70,6 +70,7 @@ class RoundConfig(TypedDict): session_id: str helper_type: str aggregator: str + client_config: dict class RoundHandler: diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index e33a0bba0..7926ab0a8 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: network/grpc/fedn.proto +# source: fedn/network/grpc/fedn.proto # Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor @@ -15,79 +15,109 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xdb\x01\n\x0fModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x15\n\rprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x8b\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\x14\n\x10MODEL_PREDICTION\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfd\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12<\n\x13SendModelPrediction\x12\x15.fedn.ModelPrediction\x1a\x0e.fedn.Responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xdb\x01\n\x0fModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x15\n\rprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"1\n\x18ProvidedFunctionsRequest\x12\x15\n\rfunction_code\x18\x01 \x01(\t\"\xac\x01\n\x19ProvidedFunctionsResponse\x12T\n\x13\x61vailable_functions\x18\x01 \x03(\x0b\x32\x37.fedn.ProvidedFunctionsResponse.AvailableFunctionsEntry\x1a\x39\n\x17\x41vailableFunctionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"#\n\x13\x43lientConfigRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"-\n\x14\x43lientConfigResponse\x12\x15\n\rclient_config\x18\x01 \x01(\t\",\n\x16\x43lientSelectionRequest\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"-\n\x17\x43lientSelectionResponse\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"8\n\x11\x43lientMetaRequest\x12\x10\n\x08metadata\x18\x01 \x01(\t\x12\x11\n\tclient_id\x18\x02 \x01(\t\"$\n\x12\x43lientMetaResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"-\n\x11StoreModelRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\"$\n\x12StoreModelResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"\'\n\x12\x41ggregationRequest\x12\x11\n\taggregate\x18\x01 \x01(\t\"#\n\x13\x41ggregationResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c*\x8b\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\x14\n\x10MODEL_PREDICTION\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbb\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x41\n\x12SetServerFunctions\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfd\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12<\n\x13SendModelPrediction\x12\x15.fedn.ModelPrediction\x1a\x0e.fedn.Response2\xec\x03\n\x0f\x46unctionService\x12Z\n\x17HandleProvidedFunctions\x12\x1e.fedn.ProvidedFunctionsRequest\x1a\x1f.fedn.ProvidedFunctionsResponse\x12M\n\x12HandleClientConfig\x12\x19.fedn.ClientConfigRequest\x1a\x1a.fedn.ClientConfigResponse(\x01\x12T\n\x15HandleClientSelection\x12\x1c.fedn.ClientSelectionRequest\x1a\x1d.fedn.ClientSelectionResponse\x12\x43\n\x0eHandleMetadata\x12\x17.fedn.ClientMetaRequest\x1a\x18.fedn.ClientMetaResponse\x12G\n\x10HandleStoreModel\x12\x17.fedn.StoreModelRequest\x1a\x18.fedn.StoreModelResponse(\x01\x12J\n\x11HandleAggregation\x12\x18.fedn.AggregationRequest\x1a\x19.fedn.AggregationResponse0\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'fedn.network.grpc.fedn_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_STATUSTYPE']._serialized_start=2549 - _globals['_STATUSTYPE']._serialized_end=2688 - _globals['_QUEUE']._serialized_start=2690 - _globals['_QUEUE']._serialized_end=2726 - _globals['_MODELSTATUS']._serialized_start=2728 - _globals['_MODELSTATUS']._serialized_end=2811 - _globals['_ROLE']._serialized_start=2813 - _globals['_ROLE']._serialized_end=2869 - _globals['_COMMAND']._serialized_start=2871 - _globals['_COMMAND']._serialized_end=2945 - _globals['_CONNECTIONSTATUS']._serialized_start=2947 - _globals['_CONNECTIONSTATUS']._serialized_end=3020 - _globals['_RESPONSE']._serialized_start=66 - _globals['_RESPONSE']._serialized_end=124 - _globals['_STATUS']._serialized_start=127 - _globals['_STATUS']._serialized_end=443 - _globals['_STATUS_LOGLEVEL']._serialized_start=377 - _globals['_STATUS_LOGLEVEL']._serialized_end=443 - _globals['_TASKREQUEST']._serialized_start=446 - _globals['_TASKREQUEST']._serialized_end=662 - _globals['_MODELUPDATE']._serialized_start=665 - _globals['_MODELUPDATE']._serialized_end=856 - _globals['_MODELVALIDATION']._serialized_start=859 - _globals['_MODELVALIDATION']._serialized_end=1075 - _globals['_MODELPREDICTION']._serialized_start=1078 - _globals['_MODELPREDICTION']._serialized_end=1297 - _globals['_MODELREQUEST']._serialized_start=1300 - _globals['_MODELREQUEST']._serialized_end=1437 - _globals['_MODELRESPONSE']._serialized_start=1439 - _globals['_MODELRESPONSE']._serialized_end=1532 - _globals['_GETGLOBALMODELREQUEST']._serialized_start=1534 - _globals['_GETGLOBALMODELREQUEST']._serialized_end=1619 - _globals['_GETGLOBALMODELRESPONSE']._serialized_start=1621 - _globals['_GETGLOBALMODELRESPONSE']._serialized_end=1725 - _globals['_HEARTBEAT']._serialized_start=1727 - _globals['_HEARTBEAT']._serialized_end=1768 - _globals['_CLIENTAVAILABLEMESSAGE']._serialized_start=1770 - _globals['_CLIENTAVAILABLEMESSAGE']._serialized_end=1857 - _globals['_LISTCLIENTSREQUEST']._serialized_start=1859 - _globals['_LISTCLIENTSREQUEST']._serialized_end=1939 - _globals['_CLIENTLIST']._serialized_start=1941 - _globals['_CLIENTLIST']._serialized_end=1983 - _globals['_CLIENT']._serialized_start=1985 - _globals['_CLIENT']._serialized_end=2052 - _globals['_REASSIGNREQUEST']._serialized_start=2054 - _globals['_REASSIGNREQUEST']._serialized_end=2163 - _globals['_RECONNECTREQUEST']._serialized_start=2165 - _globals['_RECONNECTREQUEST']._serialized_end=2264 - _globals['_PARAMETER']._serialized_start=2266 - _globals['_PARAMETER']._serialized_end=2305 - _globals['_CONTROLREQUEST']._serialized_start=2307 - _globals['_CONTROLREQUEST']._serialized_end=2391 - _globals['_CONTROLRESPONSE']._serialized_start=2393 - _globals['_CONTROLRESPONSE']._serialized_end=2463 - _globals['_CONNECTIONREQUEST']._serialized_start=2465 - _globals['_CONNECTIONREQUEST']._serialized_end=2484 - _globals['_CONNECTIONRESPONSE']._serialized_start=2486 - _globals['_CONNECTIONRESPONSE']._serialized_end=2546 - _globals['_MODELSERVICE']._serialized_start=3022 - _globals['_MODELSERVICE']._serialized_end=3144 - _globals['_CONTROL']._serialized_start=3147 - _globals['_CONTROL']._serialized_end=3395 - _globals['_REDUCER']._serialized_start=3397 - _globals['_REDUCER']._serialized_end=3483 - _globals['_CONNECTOR']._serialized_start=3486 - _globals['_CONNECTOR']._serialized_end=3913 - _globals['_COMBINER']._serialized_start=3916 - _globals['_COMBINER']._serialized_end=4169 + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._options = None + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_options = b'8\001' + _globals['_STATUSTYPE']._serialized_start=3216 + _globals['_STATUSTYPE']._serialized_end=3355 + _globals['_QUEUE']._serialized_start=3357 + _globals['_QUEUE']._serialized_end=3393 + _globals['_MODELSTATUS']._serialized_start=3395 + _globals['_MODELSTATUS']._serialized_end=3478 + _globals['_ROLE']._serialized_start=3480 + _globals['_ROLE']._serialized_end=3536 + _globals['_COMMAND']._serialized_start=3538 + _globals['_COMMAND']._serialized_end=3612 + _globals['_CONNECTIONSTATUS']._serialized_start=3614 + _globals['_CONNECTIONSTATUS']._serialized_end=3687 + _globals['_RESPONSE']._serialized_start=71 + _globals['_RESPONSE']._serialized_end=129 + _globals['_STATUS']._serialized_start=132 + _globals['_STATUS']._serialized_end=448 + _globals['_STATUS_LOGLEVEL']._serialized_start=382 + _globals['_STATUS_LOGLEVEL']._serialized_end=448 + _globals['_TASKREQUEST']._serialized_start=451 + _globals['_TASKREQUEST']._serialized_end=667 + _globals['_MODELUPDATE']._serialized_start=670 + _globals['_MODELUPDATE']._serialized_end=861 + _globals['_MODELVALIDATION']._serialized_start=864 + _globals['_MODELVALIDATION']._serialized_end=1080 + _globals['_MODELPREDICTION']._serialized_start=1083 + _globals['_MODELPREDICTION']._serialized_end=1302 + _globals['_MODELREQUEST']._serialized_start=1305 + _globals['_MODELREQUEST']._serialized_end=1442 + _globals['_MODELRESPONSE']._serialized_start=1444 + _globals['_MODELRESPONSE']._serialized_end=1537 + _globals['_GETGLOBALMODELREQUEST']._serialized_start=1539 + _globals['_GETGLOBALMODELREQUEST']._serialized_end=1624 + _globals['_GETGLOBALMODELRESPONSE']._serialized_start=1626 + _globals['_GETGLOBALMODELRESPONSE']._serialized_end=1730 + _globals['_HEARTBEAT']._serialized_start=1732 + _globals['_HEARTBEAT']._serialized_end=1773 + _globals['_CLIENTAVAILABLEMESSAGE']._serialized_start=1775 + _globals['_CLIENTAVAILABLEMESSAGE']._serialized_end=1862 + _globals['_LISTCLIENTSREQUEST']._serialized_start=1864 + _globals['_LISTCLIENTSREQUEST']._serialized_end=1944 + _globals['_CLIENTLIST']._serialized_start=1946 + _globals['_CLIENTLIST']._serialized_end=1988 + _globals['_CLIENT']._serialized_start=1990 + _globals['_CLIENT']._serialized_end=2057 + _globals['_REASSIGNREQUEST']._serialized_start=2059 + _globals['_REASSIGNREQUEST']._serialized_end=2168 + _globals['_RECONNECTREQUEST']._serialized_start=2170 + _globals['_RECONNECTREQUEST']._serialized_end=2269 + _globals['_PARAMETER']._serialized_start=2271 + _globals['_PARAMETER']._serialized_end=2310 + _globals['_CONTROLREQUEST']._serialized_start=2312 + _globals['_CONTROLREQUEST']._serialized_end=2396 + _globals['_CONTROLRESPONSE']._serialized_start=2398 + _globals['_CONTROLRESPONSE']._serialized_end=2468 + _globals['_CONNECTIONREQUEST']._serialized_start=2470 + _globals['_CONNECTIONREQUEST']._serialized_end=2489 + _globals['_CONNECTIONRESPONSE']._serialized_start=2491 + _globals['_CONNECTIONRESPONSE']._serialized_end=2551 + _globals['_PROVIDEDFUNCTIONSREQUEST']._serialized_start=2553 + _globals['_PROVIDEDFUNCTIONSREQUEST']._serialized_end=2602 + _globals['_PROVIDEDFUNCTIONSRESPONSE']._serialized_start=2605 + _globals['_PROVIDEDFUNCTIONSRESPONSE']._serialized_end=2777 + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_start=2720 + _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_end=2777 + _globals['_CLIENTCONFIGREQUEST']._serialized_start=2779 + _globals['_CLIENTCONFIGREQUEST']._serialized_end=2814 + _globals['_CLIENTCONFIGRESPONSE']._serialized_start=2816 + _globals['_CLIENTCONFIGRESPONSE']._serialized_end=2861 + _globals['_CLIENTSELECTIONREQUEST']._serialized_start=2863 + _globals['_CLIENTSELECTIONREQUEST']._serialized_end=2907 + _globals['_CLIENTSELECTIONRESPONSE']._serialized_start=2909 + _globals['_CLIENTSELECTIONRESPONSE']._serialized_end=2954 + _globals['_CLIENTMETAREQUEST']._serialized_start=2956 + _globals['_CLIENTMETAREQUEST']._serialized_end=3012 + _globals['_CLIENTMETARESPONSE']._serialized_start=3014 + _globals['_CLIENTMETARESPONSE']._serialized_end=3050 + _globals['_STOREMODELREQUEST']._serialized_start=3052 + _globals['_STOREMODELREQUEST']._serialized_end=3097 + _globals['_STOREMODELRESPONSE']._serialized_start=3099 + _globals['_STOREMODELRESPONSE']._serialized_end=3135 + _globals['_AGGREGATIONREQUEST']._serialized_start=3137 + _globals['_AGGREGATIONREQUEST']._serialized_end=3176 + _globals['_AGGREGATIONRESPONSE']._serialized_start=3178 + _globals['_AGGREGATIONRESPONSE']._serialized_end=3213 + _globals['_MODELSERVICE']._serialized_start=3689 + _globals['_MODELSERVICE']._serialized_end=3811 + _globals['_CONTROL']._serialized_start=3814 + _globals['_CONTROL']._serialized_end=4129 + _globals['_REDUCER']._serialized_start=4131 + _globals['_REDUCER']._serialized_end=4217 + _globals['_CONNECTOR']._serialized_start=4220 + _globals['_CONNECTOR']._serialized_end=4647 + _globals['_COMBINER']._serialized_start=4650 + _globals['_COMBINER']._serialized_end=4903 + _globals['_FUNCTIONSERVICE']._serialized_start=4906 + _globals['_FUNCTIONSERVICE']._serialized_end=5398 # @@protoc_insertion_point(module_scope) diff --git a/fedn/network/grpc/fedn_pb2_grpc.py b/fedn/network/grpc/fedn_pb2_grpc.py index d8ef6aab4..b67e7e095 100644 --- a/fedn/network/grpc/fedn_pb2_grpc.py +++ b/fedn/network/grpc/fedn_pb2_grpc.py @@ -643,8 +643,8 @@ def __init__(self, channel): ) self.SendModelPrediction = channel.unary_unary( '/fedn.Combiner/SendModelPrediction', - request_serializer=network_dot_grpc_dot_fedn__pb2.ModelPrediction.SerializeToString, - response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + request_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelPrediction.SerializeToString, + response_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, ) @@ -696,8 +696,8 @@ def add_CombinerServicer_to_server(servicer, server): ), 'SendModelPrediction': grpc.unary_unary_rpc_method_handler( servicer.SendModelPrediction, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelPrediction.FromString, - response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + request_deserializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelPrediction.FromString, + response_serializer=fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -760,6 +760,23 @@ def SendModelValidation(request, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod + def SendModelPrediction(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/fedn.Combiner/SendModelPrediction', + fedn_dot_network_dot_grpc_dot_fedn__pb2.ModelPrediction.SerializeToString, + fedn_dot_network_dot_grpc_dot_fedn__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + class FunctionServiceStub(object): """Missing associated documentation comment in .proto file.""" @@ -985,20 +1002,3 @@ def HandleAggregation(request, fedn_dot_network_dot_grpc_dot_fedn__pb2.AggregationResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) - - @staticmethod - def SendModelPrediction(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/fedn.Combiner/SendModelPrediction', - network_dot_grpc_dot_fedn__pb2.ModelPrediction.SerializeToString, - network_dot_grpc_dot_fedn__pb2.Response.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From c1af873d471fd04f74bb060f2b2d91a3d717bd85 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Wed, 30 Oct 2024 13:41:23 +0100 Subject: [PATCH 20/21] grpc improvements etc --- examples/server-functions/client/train.py | 8 +- examples/server-functions/server_functions.py | 4 +- fedn/network/api/interface.py | 5 +- fedn/network/clients/client.py | 8 +- fedn/network/combiner/aggregators/fedavg.py | 5 +- fedn/network/combiner/aggregators/fedopt.py | 6 +- .../combiner/aggregators/tests/test_fedavg.py | 2 +- fedn/network/combiner/hooks/hook_client.py | 25 ++++-- fedn/network/combiner/hooks/hooks.py | 75 ++++++++++------- .../combiner/hooks/serverfunctionsbase.py | 2 +- fedn/network/combiner/roundhandler.py | 21 +++-- fedn/network/combiner/updatehandler.py | 22 +++-- fedn/network/grpc/fedn.proto | 2 +- fedn/network/grpc/fedn_pb2.py | 84 +++++++++---------- 14 files changed, 154 insertions(+), 115 deletions(-) diff --git a/examples/server-functions/client/train.py b/examples/server-functions/client/train.py index 7d87e680e..c67d3ec69 100644 --- a/examples/server-functions/client/train.py +++ b/examples/server-functions/client/train.py @@ -11,8 +11,8 @@ # swap this to the load_metadata from helpers.helpers on release.. -def load_client_config(filename): - """Load client_config from file. +def load_client_settings(filename): + """Load client settings from file. :param filename: The name of the file to load from. :type filename: str @@ -68,8 +68,8 @@ def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1 # Load parmeters and initialize model model = load_parameters(in_model_path) - client_config = load_client_config(in_model_path) - lr = client_config["learning_rate"] + client_settings = load_client_settings(in_model_path) + lr = client_settings["learning_rate"] # Train optimizer = torch.optim.SGD(model.parameters(), lr=lr) diff --git a/examples/server-functions/server_functions.py b/examples/server-functions/server_functions.py index d8cf080a8..e07ad83ad 100644 --- a/examples/server-functions/server_functions.py +++ b/examples/server-functions/server_functions.py @@ -20,11 +20,11 @@ def client_selection(self, client_ids: List[str]) -> List: return client_ids # Called secondly before sending the global model. - def client_config(self, global_model: List[np.ndarray]) -> dict: + def client_settings(self, global_model: List[np.ndarray]) -> dict: # Decrease learning rate every 10 rounds if self.round % 10 == 0: self.lr = self.lr * 0.1 - # see client/train.py for how to load the client config. + # see client/train.py for how to load the client settings. self.round += 1 return {"learning_rate": self.lr} diff --git a/fedn/network/api/interface.py b/fedn/network/api/interface.py index 21425f897..eeaff3e05 100644 --- a/fedn/network/api/interface.py +++ b/fedn/network/api/interface.py @@ -10,7 +10,6 @@ from fedn.common.config import FEDN_COMPUTE_PACKAGE_DIR, get_controller_config, get_network_config from fedn.common.log_config import logger from fedn.network.combiner.interfaces import CombinerUnavailableError -from fedn.network.combiner.modelservice import load_model_from_bytes from fedn.network.state import ReducerState, ReducerStateToString from fedn.utils.checksum import sha @@ -603,6 +602,10 @@ def set_initial_model(self, file): """ logger.info("Adding model") try: + object = BytesIO() + object.seek(0, 0) + file.seek(0) + object.write(file.read()) helper = self.control.get_helper() logger.info(f"Loading model from file using helper {helper.name}") object.seek(0) diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index 96006f03d..92731ccf3 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -456,7 +456,7 @@ def _listen_to_task_stream(self): if not self._connected: return - def _process_training_request(self, model_id: str, session_id: str = None, client_config: dict = None): + def _process_training_request(self, model_id: str, session_id: str = None, client_settings: dict = None): """Process a training (model update) request. :param model_id: The model id of the model to be updated. @@ -482,7 +482,7 @@ def _process_training_request(self, model_id: str, session_id: str = None, clien with open(inpath, "wb") as fh: fh.write(mdl.getbuffer()) - save_metadata(metadata=client_config, filename=inpath) + save_metadata(metadata=client_settings, filename=inpath) outpath = self.helper.get_tmp_path() tic = time.time() @@ -615,8 +615,8 @@ def process_request(self): if task_type == "train": tic = time.time() self.state = ClientState.training - client_config = json.loads(request.data).get("client_config", {}) - model_id, meta = self._process_training_request(request.model_id, session_id=request.session_id, client_config=client_config) + client_settings = json.loads(request.data).get("client_settings", {}) + model_id, meta = self._process_training_request(request.model_id, session_id=request.session_id, client_settings=client_settings) if meta is not None: processing_time = time.time() - tic diff --git a/fedn/network/combiner/aggregators/fedavg.py b/fedn/network/combiner/aggregators/fedavg.py index 2ab155e67..71dab273a 100644 --- a/fedn/network/combiner/aggregators/fedavg.py +++ b/fedn/network/combiner/aggregators/fedavg.py @@ -43,11 +43,10 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): logger.info("AGGREGATOR({}): Aggregating model updates... ".format(self.name)) - model_updates = self.update_handler.get_model_updates() - - for model_update in model_updates: + while not self.update_handler.model_updates.empty(): try: logger.info("AGGREGATOR({}): Getting next model update from queue.".format(self.name)) + model_update = self.update_handler.next_model_update() # Load model parameters and metadata logger.info("AGGREGATOR({}): Loading model metadata {}.".format(self.name, model_update.model_update_id)) diff --git a/fedn/network/combiner/aggregators/fedopt.py b/fedn/network/combiner/aggregators/fedopt.py index ac53795e7..d91fe6d22 100644 --- a/fedn/network/combiner/aggregators/fedopt.py +++ b/fedn/network/combiner/aggregators/fedopt.py @@ -95,10 +95,10 @@ def combine_models(self, helper=None, delete_models=True, parameters=None): logger.info("AGGREGATOR({}): Aggregating model updates... ".format(self.name)) - model_updates = self.update_handler.get_model_updates() - - for model_update in model_updates: + while not self.update_handler.model_updates.empty(): try: + logger.info("AGGREGATOR({}): Getting next model update from queue.".format(self.name)) + model_update = self.update_handler.next_model_update() # Load model paratmeters and metadata model_next, metadata = self.update_handler.load_model_update(model_update, helper) diff --git a/fedn/network/combiner/aggregators/tests/test_fedavg.py b/fedn/network/combiner/aggregators/tests/test_fedavg.py index 3479bc74e..583a7f51a 100644 --- a/fedn/network/combiner/aggregators/tests/test_fedavg.py +++ b/fedn/network/combiner/aggregators/tests/test_fedavg.py @@ -18,7 +18,7 @@ def test_fedavg_init(self, *args, **kwargs): def test_fedavg_combine_models(self, *args, **kwargs): """Test the FedAvg aggregator combine_models method with mock classes and methods""" aggregator = FedAvg("id", None, None, None, None) - aggregator.update_handler.get_model_updates = MagicMock(return_value=[(None, None, None)]) + aggregator.update_handler.next_model_update = MagicMock(return_value=[(None, None, None)]) aggregator.server = MagicMock() data = {} diff --git a/fedn/network/combiner/hooks/hook_client.py b/fedn/network/combiner/hooks/hook_client.py index 8b8a9b6bb..340305881 100644 --- a/fedn/network/combiner/hooks/hook_client.py +++ b/fedn/network/combiner/hooks/hook_client.py @@ -18,7 +18,17 @@ class CombinerHookInterface: def __init__(self): """Initialize CombinerHookInterface client.""" self.hook_service_host = os.getenv("HOOK_SERVICE_HOST", "hook:12081") - self.channel = grpc.insecure_channel(self.hook_service_host) + self.channel = grpc.insecure_channel( + self.hook_service_host, + options=[ + ("grpc.keepalive_time_ms", 30000), # 30 seconds ping interval + ("grpc.keepalive_timeout_ms", 5000), # 5 seconds timeout for a response + ("grpc.keepalive_permit_without_calls", 1), # allow keepalives even with no active calls + ("grpc.enable_retries", 1), # automatic retries + ("grpc.initial_reconnect_backoff_ms", 1000), # initial delay before retrying + ("grpc.max_reconnect_backoff_ms", 5000), # maximum delay before retrying + ], + ) self.stub = rpc.FunctionServiceStub(self.channel) def provided_functions(self, server_functions: str): @@ -34,7 +44,7 @@ def provided_functions(self, server_functions: str): response = self.stub.HandleProvidedFunctions(request) return response.available_functions - def client_config(self, global_model) -> dict: + def client_settings(self, global_model) -> dict: """Communicates to hook container to get a client config. :param global_model: The global model that will be distributed to clients. @@ -46,7 +56,7 @@ def client_config(self, global_model) -> dict: args = {} model = model_as_bytesIO(global_model) response = self.stub.HandleClientConfig(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) - return json.loads(response.client_config) + return json.loads(response.client_settings) def client_selection(self, clients: list) -> list: request = fedn.ClientSelectionRequest(client_ids=json.dumps(clients)) @@ -70,8 +80,10 @@ def aggregate(self, previous_global, update_handler: UpdateHandler, helper, dele response = self.stub.HandleStoreModel(bytesIO_request_generator(mdl=previous_global, request_function=request_function, args=args)) logger.info(f"Store model response: {response.status}") # send client models and metadata - updates = update_handler.get_model_updates() - for update in updates: + nr_updates = 0 + while not update_handler.model_updates.empty(): + logger.info("Getting next model update from queue.") + update = update_handler.next_model_update() metadata = json.loads(update.meta)["training_metadata"] model = update_handler.load_model_update_bytesIO(update.model_update_id) # send metadata @@ -83,12 +95,13 @@ def aggregate(self, previous_global, update_handler: UpdateHandler, helper, dele request_function = fedn.StoreModelRequest response = self.stub.HandleStoreModel(bytesIO_request_generator(mdl=model, request_function=request_function, args=args)) logger.info(f"Store model response: {response.status}") + nr_updates += 1 if delete_models: # delete model from disk update_handler.delete_model(model_update=update) # ask for aggregation request = fedn.AggregationRequest(aggregate="aggregate") response_generator = self.stub.HandleAggregation(request) - data["nr_aggregated_models"] = len(updates) + data["nr_aggregated_models"] = nr_updates model, _ = unpack_model(response_generator, helper) return model, data diff --git a/fedn/network/combiner/hooks/hooks.py b/fedn/network/combiner/hooks/hooks.py index 666e27cc4..a20ddb27f 100644 --- a/fedn/network/combiner/hooks/hooks.py +++ b/fedn/network/combiner/hooks/hooks.py @@ -30,6 +30,7 @@ def __init__(self) -> None: self.server_functions: ServerFunctionsBase = None self.server_functions_code: str = None self.client_updates = {} + self.implemented_functions = None def HandleClientConfig(self, request_iterator: fedn.ClientConfigRequest, context): """Distribute client configs to clients from user defined code. @@ -43,9 +44,9 @@ def HandleClientConfig(self, request_iterator: fedn.ClientConfigRequest, context """ logger.info("Received client config request.") model, _ = unpack_model(request_iterator, self.helper) - client_config = self.server_functions.client_config(global_model=model) - logger.info(f"Client config response: {client_config}") - return fedn.ClientConfigResponse(client_config=json.dumps(client_config)) + client_settings = self.server_functions.client_settings(global_model=model) + logger.info(f"Client config response: {client_settings}") + return fedn.ClientConfigResponse(client_settings=json.dumps(client_settings)) def HandleClientSelection(self, request: fedn.ClientSelectionRequest, context): """Handle client selection from user defined code. @@ -121,53 +122,71 @@ def HandleProvidedFunctions(self, request: fedn.ProvidedFunctionsResponse, conte :rtype: :class:`fedn.network.grpc.fedn_pb2.ProvidedFunctionsResponse` """ logger.info("Receieved provided functions request.") + if self.implemented_functions is not None: + return fedn.ProvidedFunctionsResponse(available_functions=self.implemented_functions) server_functions_code = request.function_code - if self.server_functions is None or server_functions_code != self.server_functions_code: - self.server_functions_code = server_functions_code - # this will create a new user defined instance of the ServerFunctions class. - try: - namespace = {} - exec(server_functions_code, globals(), namespace) # noqa: S102 - exec("server_functions = ServerFunctions()", globals(), namespace) # noqa: S102 - self.server_functions = namespace.get("server_functions") - except Exception as e: - logger.error(f"Exec failed with error: {str(e)}") - implemented_functions = {} + self.server_functions_code = server_functions_code + self.implemented_functions = {} + self._instansiate_server_functions_code() # if crashed or not returning None we assume function is implemented # check if aggregation is available try: ret = self.server_functions.aggregate(0, 0) if ret is None: - implemented_functions["aggregate"] = False + self.implemented_functions["aggregate"] = False else: - implemented_functions["aggregate"] = True + self.implemented_functions["aggregate"] = True except Exception: - implemented_functions["aggregate"] = True - # check if client_config is available + self.implemented_functions["aggregate"] = True + # check if client_settings is available try: - ret = self.server_functions.client_config(0) + ret = self.server_functions.client_settings(0) if ret is None: - implemented_functions["client_config"] = False + self.implemented_functions["client_settings"] = False else: - implemented_functions["client_config"] = True + self.implemented_functions["client_settings"] = True except Exception: - implemented_functions["client_config"] = True + self.implemented_functions["client_settings"] = True # check if client_selection is available try: ret = self.server_functions.client_selection(0) if ret is None: - implemented_functions["client_selection"] = False + self.implemented_functions["client_selection"] = False else: - implemented_functions["client_selection"] = True + self.implemented_functions["client_selection"] = True except Exception: - implemented_functions["client_selection"] = True - logger.info(f"Provided function: {implemented_functions}") - return fedn.ProvidedFunctionsResponse(available_functions=implemented_functions) + self.implemented_functions["client_selection"] = True + logger.info(f"Provided function: {self.implemented_functions}") + return fedn.ProvidedFunctionsResponse(available_functions=self.implemented_functions) + + def _instansiate_server_functions_code(self): + # this will create a new user defined instance of the ServerFunctions class. + try: + namespace = {} + exec(self.server_functions_code, globals(), namespace) # noqa: S102 + exec("server_functions = ServerFunctions()", globals(), namespace) # noqa: S102 + self.server_functions = namespace.get("server_functions") + except Exception as e: + logger.error(f"Exec failed with error: {str(e)}") def serve(): """Start the hooks service.""" - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + # Keepalive settings: these detect if the client is alive + KEEPALIVE_TIME_MS = 60 * 1000 # send keepalive ping every 60 seconds + KEEPALIVE_TIMEOUT_MS = 20 * 1000 # wait 20 seconds for keepalive ping ack before considering connection dead + MAX_CONNECTION_IDLE_MS = 5 * 60 * 1000 # max idle time before server terminates the connection (5 minutes) + MAX_MESSAGE_LENGTH = 1 * 1024 * 1024 * 1024 # 1 GB in bytes + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=100), # Increase based on expected load + options=[ + ("grpc.keepalive_time_ms", KEEPALIVE_TIME_MS), + ("grpc.keepalive_timeout_ms", KEEPALIVE_TIMEOUT_MS), + ("grpc.max_connection_idle_ms", MAX_CONNECTION_IDLE_MS), + ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH), + ("grpc.max_receive_message_length", -1), + ], + ) rpc.add_FunctionServiceServicer_to_server(FunctionServiceServicer(), server) server.add_insecure_port("[::]:12081") server.start() diff --git a/fedn/network/combiner/hooks/serverfunctionsbase.py b/fedn/network/combiner/hooks/serverfunctionsbase.py index c276941d7..6b0666164 100644 --- a/fedn/network/combiner/hooks/serverfunctionsbase.py +++ b/fedn/network/combiner/hooks/serverfunctionsbase.py @@ -35,7 +35,7 @@ def aggregate(self, previous_global: List[np.ndarray], client_updates: Dict[str, """ pass - def client_config(self, global_model: List[np.ndarray]) -> Dict: + def client_settings(self, global_model: List[np.ndarray]) -> Dict: """Returns metadata related to the model, which gets distributed to the clients. The dictionary may only contain primitive types. diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 050794fd5..5eb5387d8 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -51,8 +51,8 @@ class RoundConfig(TypedDict): :type helper_type: str :param aggregator: The aggregator type. :type aggregator: str - :param client_config: Configs that are distributed to clients. - :type client_config: dict + :param client_settings: Settings that are distributed to clients. + :type client_settings: dict """ _job_id: str @@ -70,7 +70,7 @@ class RoundConfig(TypedDict): session_id: str helper_type: str aggregator: str - client_config: dict + client_settings: dict class RoundHandler: @@ -97,6 +97,7 @@ def __init__(self, server): self.modelservice = modelservice self.server_functions = inspect.getsource(ServerFunctions) self.update_handler = UpdateHandler(modelservice=modelservice) + self.hook_interface = CombinerHookInterface() def set_aggregator(self, aggregator): self.aggregator = get_aggregator(aggregator, self.update_handler) @@ -140,10 +141,10 @@ def _training_round(self, config, clients, provided_functions): session_id = config["session_id"] model_id = config["model_id"] - if provided_functions["client_config"]: + if provided_functions["client_settings"]: global_model_bytes = self.modelservice.temp_model_storage.get(model_id) - client_config = CombinerHookInterface().client_config(global_model_bytes) - config["client_config"] = client_config + client_settings = self.hook_interface.client_settings(global_model_bytes) + config["client_settings"] = client_settings # Request model updates from all active clients. self.server.request_model_update(session_id=session_id, model_id=model_id, config=config, clients=clients) @@ -173,15 +174,13 @@ def _training_round(self, config, clients, provided_functions): parameters = None if provided_functions["aggregate"]: previous_model_bytes = self.modelservice.temp_model_storage.get(model_id) - model, data = CombinerHookInterface().aggregate(previous_model_bytes, self.update_handler, helper, delete_models=delete_models) + model, data = self.hook_interface.aggregate(previous_model_bytes, self.update_handler, helper, delete_models=delete_models) else: model, data = self.aggregator.combine_models(helper=helper, delete_models=delete_models, parameters=parameters) except Exception as e: logger.warning("AGGREGATION FAILED AT COMBINER! {}".format(e)) raise - self.update_handler.flush() - meta["time_combination"] = time.time() - tic meta["aggregation_time"] = data return model, meta @@ -325,10 +324,10 @@ def execute_training_round(self, config): # Download model to update and set in temp storage. self.stage_model(config["model_id"]) - provided_functions = CombinerHookInterface().provided_functions(self.server_functions) + provided_functions = self.hook_interface.provided_functions(self.server_functions) if provided_functions["client_selection"]: - clients = CombinerHookInterface().client_selection(clients=self.server.get_active_trainers()) + clients = self.hook_interface.client_selection(clients=self.server.get_active_trainers()) else: clients = self._assign_round_clients(self.server.max_clients) model, meta = self._training_round(config, clients, provided_functions) diff --git a/fedn/network/combiner/updatehandler.py b/fedn/network/combiner/updatehandler.py index 08e5696a1..517595d13 100644 --- a/fedn/network/combiner/updatehandler.py +++ b/fedn/network/combiner/updatehandler.py @@ -1,4 +1,5 @@ import json +import queue import sys import time import traceback @@ -21,7 +22,7 @@ class UpdateHandler: """ def __init__(self, modelservice: ModelService) -> None: - self.model_updates = [] + self.model_updates = queue.Queue() self.modelservice = modelservice self.model_id_to_model_data = {} @@ -30,8 +31,16 @@ def delete_model(self, model_update): self.modelservice.temp_model_storage.delete(model_update.model_update_id) logger.info("UPDATE HANDLER: Deleted model update {} from storage.".format(model_update.model_update_id)) - def get_model_updates(self): - return self.model_updates + def next_model_update(self): + """Get the next model update from the queue. + + :param helper: A helper object. + :type helper: object + :return: The model update. + :rtype: fedn.network.grpc.fedn.proto.ModelUpdate + """ + model_update = self.model_updates.get(block=False) + return model_update def on_model_update(self, model_update): """Callback when a new client model update is recieved. @@ -50,7 +59,7 @@ def on_model_update(self, model_update): valid_update = self._validate_model_update(model_update) if valid_update: # Push the model update to the processing queue - self.model_updates.append(model_update) + self.model_updates.put(model_update) else: logger.warning("UPDATE HANDLER: Invalid model update, skipping.") except Exception as e: @@ -194,11 +203,8 @@ def waitforit(self, config, buffer_size=100, polling_interval=0.1): tt = 0.0 while tt < time_window: - if len(self.model_updates) >= buffer_size: + if self.model_updates.qsize() >= buffer_size: break time.sleep(polling_interval) tt += polling_interval - - def flush(self): - self.model_updates = [] diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index 647157fd8..fb7c312f4 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -270,7 +270,7 @@ message ClientConfigRequest { } message ClientConfigResponse { - string client_config = 1; + string client_settings = 1; } message ClientSelectionRequest { diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index 7926ab0a8..09ecdec36 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -15,7 +15,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xdb\x01\n\x0fModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x15\n\rprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"1\n\x18ProvidedFunctionsRequest\x12\x15\n\rfunction_code\x18\x01 \x01(\t\"\xac\x01\n\x19ProvidedFunctionsResponse\x12T\n\x13\x61vailable_functions\x18\x01 \x03(\x0b\x32\x37.fedn.ProvidedFunctionsResponse.AvailableFunctionsEntry\x1a\x39\n\x17\x41vailableFunctionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"#\n\x13\x43lientConfigRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"-\n\x14\x43lientConfigResponse\x12\x15\n\rclient_config\x18\x01 \x01(\t\",\n\x16\x43lientSelectionRequest\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"-\n\x17\x43lientSelectionResponse\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"8\n\x11\x43lientMetaRequest\x12\x10\n\x08metadata\x18\x01 \x01(\t\x12\x11\n\tclient_id\x18\x02 \x01(\t\"$\n\x12\x43lientMetaResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"-\n\x11StoreModelRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\"$\n\x12StoreModelResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"\'\n\x12\x41ggregationRequest\x12\x11\n\taggregate\x18\x01 \x01(\t\"#\n\x13\x41ggregationResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c*\x8b\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\x14\n\x10MODEL_PREDICTION\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbb\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x41\n\x12SetServerFunctions\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfd\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12<\n\x13SendModelPrediction\x12\x15.fedn.ModelPrediction\x1a\x0e.fedn.Response2\xec\x03\n\x0f\x46unctionService\x12Z\n\x17HandleProvidedFunctions\x12\x1e.fedn.ProvidedFunctionsRequest\x1a\x1f.fedn.ProvidedFunctionsResponse\x12M\n\x12HandleClientConfig\x12\x19.fedn.ClientConfigRequest\x1a\x1a.fedn.ClientConfigResponse(\x01\x12T\n\x15HandleClientSelection\x12\x1c.fedn.ClientSelectionRequest\x1a\x1d.fedn.ClientSelectionResponse\x12\x43\n\x0eHandleMetadata\x12\x17.fedn.ClientMetaRequest\x1a\x18.fedn.ClientMetaResponse\x12G\n\x10HandleStoreModel\x12\x17.fedn.StoreModelRequest\x1a\x18.fedn.StoreModelResponse(\x01\x12J\n\x11HandleAggregation\x12\x18.fedn.AggregationRequest\x1a\x19.fedn.AggregationResponse0\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x64n/network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xdb\x01\n\x0fModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x15\n\rprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus\"1\n\x18ProvidedFunctionsRequest\x12\x15\n\rfunction_code\x18\x01 \x01(\t\"\xac\x01\n\x19ProvidedFunctionsResponse\x12T\n\x13\x61vailable_functions\x18\x01 \x03(\x0b\x32\x37.fedn.ProvidedFunctionsResponse.AvailableFunctionsEntry\x1a\x39\n\x17\x41vailableFunctionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"#\n\x13\x43lientConfigRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"/\n\x14\x43lientConfigResponse\x12\x17\n\x0f\x63lient_settings\x18\x01 \x01(\t\",\n\x16\x43lientSelectionRequest\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"-\n\x17\x43lientSelectionResponse\x12\x12\n\nclient_ids\x18\x01 \x01(\t\"8\n\x11\x43lientMetaRequest\x12\x10\n\x08metadata\x18\x01 \x01(\t\x12\x11\n\tclient_id\x18\x02 \x01(\t\"$\n\x12\x43lientMetaResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"-\n\x11StoreModelRequest\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\"$\n\x12StoreModelResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"\'\n\x12\x41ggregationRequest\x12\x11\n\taggregate\x18\x01 \x01(\t\"#\n\x13\x41ggregationResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c*\x8b\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\x14\n\x10MODEL_PREDICTION\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xbb\x02\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x41\n\x12SetServerFunctions\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfd\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12<\n\x13SendModelPrediction\x12\x15.fedn.ModelPrediction\x1a\x0e.fedn.Response2\xec\x03\n\x0f\x46unctionService\x12Z\n\x17HandleProvidedFunctions\x12\x1e.fedn.ProvidedFunctionsRequest\x1a\x1f.fedn.ProvidedFunctionsResponse\x12M\n\x12HandleClientConfig\x12\x19.fedn.ClientConfigRequest\x1a\x1a.fedn.ClientConfigResponse(\x01\x12T\n\x15HandleClientSelection\x12\x1c.fedn.ClientSelectionRequest\x1a\x1d.fedn.ClientSelectionResponse\x12\x43\n\x0eHandleMetadata\x12\x17.fedn.ClientMetaRequest\x1a\x18.fedn.ClientMetaResponse\x12G\n\x10HandleStoreModel\x12\x17.fedn.StoreModelRequest\x1a\x18.fedn.StoreModelResponse(\x01\x12J\n\x11HandleAggregation\x12\x18.fedn.AggregationRequest\x1a\x19.fedn.AggregationResponse0\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -24,18 +24,18 @@ DESCRIPTOR._options = None _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._options = None _globals['_PROVIDEDFUNCTIONSRESPONSE_AVAILABLEFUNCTIONSENTRY']._serialized_options = b'8\001' - _globals['_STATUSTYPE']._serialized_start=3216 - _globals['_STATUSTYPE']._serialized_end=3355 - _globals['_QUEUE']._serialized_start=3357 - _globals['_QUEUE']._serialized_end=3393 - _globals['_MODELSTATUS']._serialized_start=3395 - _globals['_MODELSTATUS']._serialized_end=3478 - _globals['_ROLE']._serialized_start=3480 - _globals['_ROLE']._serialized_end=3536 - _globals['_COMMAND']._serialized_start=3538 - _globals['_COMMAND']._serialized_end=3612 - _globals['_CONNECTIONSTATUS']._serialized_start=3614 - _globals['_CONNECTIONSTATUS']._serialized_end=3687 + _globals['_STATUSTYPE']._serialized_start=3218 + _globals['_STATUSTYPE']._serialized_end=3357 + _globals['_QUEUE']._serialized_start=3359 + _globals['_QUEUE']._serialized_end=3395 + _globals['_MODELSTATUS']._serialized_start=3397 + _globals['_MODELSTATUS']._serialized_end=3480 + _globals['_ROLE']._serialized_start=3482 + _globals['_ROLE']._serialized_end=3538 + _globals['_COMMAND']._serialized_start=3540 + _globals['_COMMAND']._serialized_end=3614 + _globals['_CONNECTIONSTATUS']._serialized_start=3616 + _globals['_CONNECTIONSTATUS']._serialized_end=3689 _globals['_RESPONSE']._serialized_start=71 _globals['_RESPONSE']._serialized_end=129 _globals['_STATUS']._serialized_start=132 @@ -91,33 +91,33 @@ _globals['_CLIENTCONFIGREQUEST']._serialized_start=2779 _globals['_CLIENTCONFIGREQUEST']._serialized_end=2814 _globals['_CLIENTCONFIGRESPONSE']._serialized_start=2816 - _globals['_CLIENTCONFIGRESPONSE']._serialized_end=2861 - _globals['_CLIENTSELECTIONREQUEST']._serialized_start=2863 - _globals['_CLIENTSELECTIONREQUEST']._serialized_end=2907 - _globals['_CLIENTSELECTIONRESPONSE']._serialized_start=2909 - _globals['_CLIENTSELECTIONRESPONSE']._serialized_end=2954 - _globals['_CLIENTMETAREQUEST']._serialized_start=2956 - _globals['_CLIENTMETAREQUEST']._serialized_end=3012 - _globals['_CLIENTMETARESPONSE']._serialized_start=3014 - _globals['_CLIENTMETARESPONSE']._serialized_end=3050 - _globals['_STOREMODELREQUEST']._serialized_start=3052 - _globals['_STOREMODELREQUEST']._serialized_end=3097 - _globals['_STOREMODELRESPONSE']._serialized_start=3099 - _globals['_STOREMODELRESPONSE']._serialized_end=3135 - _globals['_AGGREGATIONREQUEST']._serialized_start=3137 - _globals['_AGGREGATIONREQUEST']._serialized_end=3176 - _globals['_AGGREGATIONRESPONSE']._serialized_start=3178 - _globals['_AGGREGATIONRESPONSE']._serialized_end=3213 - _globals['_MODELSERVICE']._serialized_start=3689 - _globals['_MODELSERVICE']._serialized_end=3811 - _globals['_CONTROL']._serialized_start=3814 - _globals['_CONTROL']._serialized_end=4129 - _globals['_REDUCER']._serialized_start=4131 - _globals['_REDUCER']._serialized_end=4217 - _globals['_CONNECTOR']._serialized_start=4220 - _globals['_CONNECTOR']._serialized_end=4647 - _globals['_COMBINER']._serialized_start=4650 - _globals['_COMBINER']._serialized_end=4903 - _globals['_FUNCTIONSERVICE']._serialized_start=4906 - _globals['_FUNCTIONSERVICE']._serialized_end=5398 + _globals['_CLIENTCONFIGRESPONSE']._serialized_end=2863 + _globals['_CLIENTSELECTIONREQUEST']._serialized_start=2865 + _globals['_CLIENTSELECTIONREQUEST']._serialized_end=2909 + _globals['_CLIENTSELECTIONRESPONSE']._serialized_start=2911 + _globals['_CLIENTSELECTIONRESPONSE']._serialized_end=2956 + _globals['_CLIENTMETAREQUEST']._serialized_start=2958 + _globals['_CLIENTMETAREQUEST']._serialized_end=3014 + _globals['_CLIENTMETARESPONSE']._serialized_start=3016 + _globals['_CLIENTMETARESPONSE']._serialized_end=3052 + _globals['_STOREMODELREQUEST']._serialized_start=3054 + _globals['_STOREMODELREQUEST']._serialized_end=3099 + _globals['_STOREMODELRESPONSE']._serialized_start=3101 + _globals['_STOREMODELRESPONSE']._serialized_end=3137 + _globals['_AGGREGATIONREQUEST']._serialized_start=3139 + _globals['_AGGREGATIONREQUEST']._serialized_end=3178 + _globals['_AGGREGATIONRESPONSE']._serialized_start=3180 + _globals['_AGGREGATIONRESPONSE']._serialized_end=3215 + _globals['_MODELSERVICE']._serialized_start=3691 + _globals['_MODELSERVICE']._serialized_end=3813 + _globals['_CONTROL']._serialized_start=3816 + _globals['_CONTROL']._serialized_end=4131 + _globals['_REDUCER']._serialized_start=4133 + _globals['_REDUCER']._serialized_end=4219 + _globals['_CONNECTOR']._serialized_start=4222 + _globals['_CONNECTOR']._serialized_end=4649 + _globals['_COMBINER']._serialized_start=4652 + _globals['_COMBINER']._serialized_end=4905 + _globals['_FUNCTIONSERVICE']._serialized_start=4908 + _globals['_FUNCTIONSERVICE']._serialized_end=5400 # @@protoc_insertion_point(module_scope) From 2df792012fc4ff61ca0bb9067fc341f68aaa07d2 Mon Sep 17 00:00:00 2001 From: viktorvaladi Date: Fri, 1 Nov 2024 16:46:29 +0100 Subject: [PATCH 21/21] fail gracefully --- fedn/network/combiner/hooks/hook_client.py | 39 +++++++++++++--------- fedn/network/combiner/roundhandler.py | 8 ++--- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/fedn/network/combiner/hooks/hook_client.py b/fedn/network/combiner/hooks/hook_client.py index 340305881..3219e5a17 100644 --- a/fedn/network/combiner/hooks/hook_client.py +++ b/fedn/network/combiner/hooks/hook_client.py @@ -17,19 +17,22 @@ class CombinerHookInterface: def __init__(self): """Initialize CombinerHookInterface client.""" - self.hook_service_host = os.getenv("HOOK_SERVICE_HOST", "hook:12081") - self.channel = grpc.insecure_channel( - self.hook_service_host, - options=[ - ("grpc.keepalive_time_ms", 30000), # 30 seconds ping interval - ("grpc.keepalive_timeout_ms", 5000), # 5 seconds timeout for a response - ("grpc.keepalive_permit_without_calls", 1), # allow keepalives even with no active calls - ("grpc.enable_retries", 1), # automatic retries - ("grpc.initial_reconnect_backoff_ms", 1000), # initial delay before retrying - ("grpc.max_reconnect_backoff_ms", 5000), # maximum delay before retrying - ], - ) - self.stub = rpc.FunctionServiceStub(self.channel) + try: + self.hook_service_host = os.getenv("HOOK_SERVICE_HOST", "hook:12081") + self.channel = grpc.insecure_channel( + self.hook_service_host, + options=[ + ("grpc.keepalive_time_ms", 30000), # 30 seconds ping interval + ("grpc.keepalive_timeout_ms", 5000), # 5 seconds timeout for a response + ("grpc.keepalive_permit_without_calls", 1), # allow keepalives even with no active calls + ("grpc.enable_retries", 1), # automatic retries + ("grpc.initial_reconnect_backoff_ms", 1000), # initial delay before retrying + ("grpc.max_reconnect_backoff_ms", 5000), # maximum delay before retrying + ], + ) + self.stub = rpc.FunctionServiceStub(self.channel) + except Exception as e: + logger.warning(f"Failed to initialize connection to hooks container with error {e}") def provided_functions(self, server_functions: str): """Communicates to hook container and asks which functions are available. @@ -39,10 +42,14 @@ def provided_functions(self, server_functions: str): :return: dictionary specifing which functions are implemented. :rtype: dict """ - request = fedn.ProvidedFunctionsRequest(function_code=server_functions) + try: + request = fedn.ProvidedFunctionsRequest(function_code=server_functions) - response = self.stub.HandleProvidedFunctions(request) - return response.available_functions + response = self.stub.HandleProvidedFunctions(request) + return response.available_functions + except Exception as e: + logger.warning(f"Was not able to communicate to hooks container due to: {e}") + return {} def client_settings(self, global_model) -> dict: """Communicates to hook container to get a client config. diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 5eb5387d8..fa3d83e8f 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -121,7 +121,7 @@ def push_round_config(self, round_config: RoundConfig) -> str: raise return round_config["_job_id"] - def _training_round(self, config, clients, provided_functions): + def _training_round(self, config: dict, clients: list, provided_functions: dict): """Send model update requests to clients and aggregate results. :param config: The round config object (passed to the client). @@ -141,7 +141,7 @@ def _training_round(self, config, clients, provided_functions): session_id = config["session_id"] model_id = config["model_id"] - if provided_functions["client_settings"]: + if provided_functions.get("client_settings", False): global_model_bytes = self.modelservice.temp_model_storage.get(model_id) client_settings = self.hook_interface.client_settings(global_model_bytes) config["client_settings"] = client_settings @@ -172,7 +172,7 @@ def _training_round(self, config, clients, provided_functions): parameters = Parameters(dict_parameters) else: parameters = None - if provided_functions["aggregate"]: + if provided_functions.get("aggregate", False): previous_model_bytes = self.modelservice.temp_model_storage.get(model_id) model, data = self.hook_interface.aggregate(previous_model_bytes, self.update_handler, helper, delete_models=delete_models) else: @@ -326,7 +326,7 @@ def execute_training_round(self, config): provided_functions = self.hook_interface.provided_functions(self.server_functions) - if provided_functions["client_selection"]: + if provided_functions.get("client_selection", False): clients = self.hook_interface.client_selection(clients=self.server.get_active_trainers()) else: clients = self._assign_round_clients(self.server.max_clients)