diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e0a059e91..6b7830bede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +## 0.9.12 (2024-08-06) + +### Bug Fixes + +- Fixed resync issue when calculating the diff of entities failed due to search identifier in relation mapping + + ## 0.9.11 (2024-08-05) diff --git a/poetry.lock b/poetry.lock index 12587a7bad..544992e177 100644 --- a/poetry.lock +++ b/poetry.lock @@ -83,33 +83,33 @@ chardet = ">=3.0.2" [[package]] name = "black" -version = "24.4.2" +version = "24.8.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce"}, - {file = "black-24.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021"}, - {file = "black-24.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063"}, - {file = "black-24.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:7768a0dbf16a39aa5e9a3ded568bb545c8c2727396d063bbaf847df05b08cd96"}, - {file = "black-24.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474"}, - {file = "black-24.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c"}, - {file = "black-24.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb"}, - {file = "black-24.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1"}, - {file = "black-24.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d"}, - {file = "black-24.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04"}, - {file = "black-24.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc"}, - {file = "black-24.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0"}, - {file = "black-24.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf10f7310db693bb62692609b397e8d67257c55f949abde4c67f9cc574492cc7"}, - {file = "black-24.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:98e123f1d5cfd42f886624d84464f7756f60ff6eab89ae845210631714f6db94"}, - {file = "black-24.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48a85f2cb5e6799a9ef05347b476cce6c182d6c71ee36925a6c194d074336ef8"}, - {file = "black-24.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:b1530ae42e9d6d5b670a34db49a94115a64596bc77710b1d05e9801e62ca0a7c"}, - {file = "black-24.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1"}, - {file = "black-24.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da33a1a5e49c4122ccdfd56cd021ff1ebc4a1ec4e2d01594fef9b6f267a9e741"}, - {file = "black-24.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e"}, - {file = "black-24.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:b9176b9832e84308818a99a561e90aa479e73c523b3f77afd07913380ae2eab7"}, - {file = "black-24.4.2-py3-none-any.whl", hash = "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c"}, - {file = "black-24.4.2.tar.gz", hash = "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d"}, + {file = "black-24.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6"}, + {file = "black-24.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb"}, + {file = "black-24.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42"}, + {file = "black-24.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a"}, + {file = "black-24.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1"}, + {file = "black-24.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af"}, + {file = "black-24.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4"}, + {file = "black-24.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af"}, + {file = "black-24.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368"}, + {file = "black-24.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed"}, + {file = "black-24.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018"}, + {file = "black-24.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2"}, + {file = "black-24.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd"}, + {file = "black-24.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2"}, + {file = "black-24.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e"}, + {file = "black-24.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920"}, + {file = "black-24.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c"}, + {file = "black-24.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e"}, + {file = "black-24.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47"}, + {file = "black-24.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb"}, + {file = "black-24.8.0-py3-none-any.whl", hash = "sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed"}, + {file = "black-24.8.0.tar.gz", hash = "sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f"}, ] [package.dependencies] @@ -420,20 +420,21 @@ all = ["email_validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)" [[package]] name = "fastapi-cli" -version = "0.0.4" +version = "0.0.5" description = "Run and manage FastAPI apps from the command line with FastAPI CLI. 🚀" optional = false python-versions = ">=3.8" files = [ - {file = "fastapi_cli-0.0.4-py3-none-any.whl", hash = "sha256:a2552f3a7ae64058cdbb530be6fa6dbfc975dc165e4fa66d224c3d396e25e809"}, - {file = "fastapi_cli-0.0.4.tar.gz", hash = "sha256:e2e9ffaffc1f7767f488d6da34b6f5a377751c996f397902eb6abb99a67bde32"}, + {file = "fastapi_cli-0.0.5-py3-none-any.whl", hash = "sha256:e94d847524648c748a5350673546bbf9bcaeb086b33c24f2e82e021436866a46"}, + {file = "fastapi_cli-0.0.5.tar.gz", hash = "sha256:d30e1239c6f46fcb95e606f02cdda59a1e2fa778a54b64686b3ff27f6211ff9f"}, ] [package.dependencies] typer = ">=0.12.3" +uvicorn = {version = ">=0.15.0", extras = ["standard"]} [package.extras] -standard = ["fastapi", "uvicorn[standard] (>=0.15.0)"] +standard = ["uvicorn[standard] (>=0.15.0)"] [[package]] name = "h11" @@ -1395,13 +1396,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.30.3" +version = "0.30.5" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.30.3-py3-none-any.whl", hash = "sha256:94a3608da0e530cea8f69683aa4126364ac18e3826b6630d1a65f4638aade503"}, - {file = "uvicorn-0.30.3.tar.gz", hash = "sha256:0d114d0831ff1adbf231d358cbf42f17333413042552a624ea6a9b4c33dcfd81"}, + {file = "uvicorn-0.30.5-py3-none-any.whl", hash = "sha256:b2d86de274726e9878188fa07576c9ceeff90a839e2b6e25c917fe05f5a6c835"}, + {file = "uvicorn-0.30.5.tar.gz", hash = "sha256:ac6fdbd4425c5fd17a9fe39daf4d4d075da6fdc80f653e5894cdc2fd98752bee"}, ] [package.dependencies] diff --git a/port_ocean/clients/port/mixins/entities.py b/port_ocean/clients/port/mixins/entities.py index 4f80d6771b..b8a510a65e 100644 --- a/port_ocean/clients/port/mixins/entities.py +++ b/port_ocean/clients/port/mixins/entities.py @@ -29,7 +29,7 @@ async def upsert_entity( request_options: RequestOptions, user_agent_type: UserAgentType | None = None, should_raise: bool = True, - ) -> None: + ) -> Entity: validation_only = request_options["validation_only"] async with self.semaphore: logger.debug( @@ -57,6 +57,12 @@ async def upsert_entity( f"blueprint: {entity.blueprint}" ) handle_status_code(response, should_raise) + result = response.json() + result_entity = Entity.parse_obj(result) + # Set the results of the search relation and identifier to the entity + entity.identifier = result_entity.identifier or entity.identifier + entity.relations = result_entity.relations or entity.relations + return entity async def batch_upsert_entities( self, @@ -64,8 +70,8 @@ async def batch_upsert_entities( request_options: RequestOptions, user_agent_type: UserAgentType | None = None, should_raise: bool = True, - ) -> None: - await asyncio.gather( + ) -> list[Entity]: + modified_entities_results = await asyncio.gather( *( self.upsert_entity( entity, @@ -75,8 +81,19 @@ async def batch_upsert_entities( ) for entity in entities ), - return_exceptions=True, + return_exceptions=should_raise, ) + entity_results = [ + entity for entity in modified_entities_results if isinstance(entity, Entity) + ] + if not should_raise: + return entity_results + + for entity_result in modified_entities_results: + if isinstance(entity_result, Exception): + raise entity_result + + return entity_results async def delete_entity( self, diff --git a/port_ocean/core/handlers/entities_state_applier/base.py b/port_ocean/core/handlers/entities_state_applier/base.py index 542c9fe77a..9956f2d6af 100644 --- a/port_ocean/core/handlers/entities_state_applier/base.py +++ b/port_ocean/core/handlers/entities_state_applier/base.py @@ -47,12 +47,15 @@ async def delete_diff( @abstractmethod async def upsert( self, entities: list[Entity], user_agent_type: UserAgentType - ) -> None: + ) -> list[Entity]: """Upsert (insert or update) the given entities into the state. Args: entities (list[Entity]): The entities to be upserted. user_agent_type (UserAgentType): The user agent responsible for the upsert. + + Returns: + list[Entity]: The upserted entities. """ pass diff --git a/port_ocean/core/handlers/entities_state_applier/port/applier.py b/port_ocean/core/handlers/entities_state_applier/port/applier.py index ae5571ce6e..dd36a243c1 100644 --- a/port_ocean/core/handlers/entities_state_applier/port/applier.py +++ b/port_ocean/core/handlers/entities_state_applier/port/applier.py @@ -71,9 +71,9 @@ async def apply_diff( logger.info( f"Updating entity diff (created: {len(diff.created)}, deleted: {len(diff.deleted)}, modified: {len(diff.modified)})" ) - await self.upsert(kept_entities, user_agent_type) + modified_entities = await self.upsert(kept_entities, user_agent_type) - await self._safe_delete(diff.deleted, kept_entities, user_agent_type) + await self._safe_delete(diff.deleted, modified_entities, user_agent_type) async def delete_diff( self, @@ -95,10 +95,11 @@ async def delete_diff( async def upsert( self, entities: list[Entity], user_agent_type: UserAgentType - ) -> None: + ) -> list[Entity]: logger.info(f"Upserting {len(entities)} entities") + modified_entities: list[Entity] = [] if event.port_app_config.create_missing_related_entities: - await self.context.port_client.batch_upsert_entities( + modified_entities = await self.context.port_client.batch_upsert_entities( entities, event.port_app_config.get_port_request_options(), user_agent_type, @@ -108,14 +109,16 @@ async def upsert( ordered_created_entities = reversed( order_by_entities_dependencies(entities) ) - for entity in ordered_created_entities: - await self.context.port_client.upsert_entity( - entity, - event.port_app_config.get_port_request_options(), - user_agent_type, - should_raise=False, + modified_entities.append( + await self.context.port_client.upsert_entity( + entity, + event.port_app_config.get_port_request_options(), + user_agent_type, + should_raise=False, + ) ) + return modified_entities async def delete( self, entities: list[Entity], user_agent_type: UserAgentType diff --git a/port_ocean/core/integrations/mixins/sync.py b/port_ocean/core/integrations/mixins/sync.py index 9154a99b74..08a13717bc 100644 --- a/port_ocean/core/integrations/mixins/sync.py +++ b/port_ocean/core/integrations/mixins/sync.py @@ -97,9 +97,11 @@ async def sync( """ entities_at_port = await ocean.port_client.search_entities(user_agent_type) - await self.entities_state_applier.upsert(entities, user_agent_type) + modified_entities = await self.entities_state_applier.upsert( + entities, user_agent_type + ) await self.entities_state_applier.delete_diff( - {"before": entities_at_port, "after": entities}, user_agent_type + {"before": entities_at_port, "after": modified_entities}, user_agent_type ) logger.info("Finished syncing change") diff --git a/port_ocean/core/integrations/mixins/sync_raw.py b/port_ocean/core/integrations/mixins/sync_raw.py index e71dc61fe4..fbcaffd956 100644 --- a/port_ocean/core/integrations/mixins/sync_raw.py +++ b/port_ocean/core/integrations/mixins/sync_raw.py @@ -140,11 +140,13 @@ async def _register_resource_raw( objects_diff = await self._calculate_raw( [(resource, results)], parse_all, send_raw_data_examples_amount ) - await self.entities_state_applier.upsert( + modified_objects = await self.entities_state_applier.upsert( objects_diff[0].entity_selector_diff.passed, user_agent_type ) - - return objects_diff[0] + return CalculationResult( + objects_diff[0].entity_selector_diff._replace(passed=modified_objects), + errors=objects_diff[0].errors, + ) async def _unregister_resource_raw( self, diff --git a/pyproject.toml b/pyproject.toml index a28c43ebf2..13a67ba1ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.9.11" +version = "0.9.12" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io"