-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
62 changed files
with
369,642 additions
and
65 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| { | ||
| "python.testing.unittestEnabled": false, | ||
| "python.testing.pytestEnabled": true, | ||
| "terminal.integrated.env.linux": { | ||
| "PYTHONPATH": ".:/home/ant/Work/Senzing/git/sz-sdk-python/src:/home/ant/Work/Senzing/git/sz-sdk-python/src/senzing:/home/ant/Work/Senzing/git/sz-sdk-python-core/src:/home/ant/Work/Senzing/git/sz-sdk-python-core/src/senzing", | ||
| // "PYTHONPATH": ".:/home/ant/Work/Senzing/git/sz-sdk-python-core/src:/home/ant/Work/Senzing/git/sz-sdk-python-core/src/senzing:./sz_tools", | ||
| "LD_LIBRARY_PATH": "/opt/senzing/er/lib/", | ||
| // "SENZING_ENGINE_CONFIGURATION_JSON": "{\"PIPELINE\":{\"CONFIGPATH\":\"/etc/opt/senzing\",\"RESOURCEPATH\":\"/opt/senzing/er/resources\",\"SUPPORTPATH\":\"/opt/senzing/data\"},\"SQL\":{\"CONNECTION\":\"sqlite3://na:na@/tmp/sqlite/G2C.db\"}}" | ||
| "SENZING_ENGINE_CONFIGURATION_JSON": "{\"PIPELINE\":{\"CONFIGPATH\":\"/etc/opt/senzing\",\"RESOURCEPATH\":\"/opt/senzing/er/resources\",\"SUPPORTPATH\":\"/opt/senzing/data\"},\"SQL\":{\"CONNECTION\":\"sqlite3://na:na@/tmp/sqlite/G2C.db\"}, \"LOGGING\":{\"CONFIG\":\"console://stdout/?style=jsonl *.TRCE;*.CRIT;*.ERR\"}}" | ||
| // "SENZING_ENGINE_CONFIGURATION_JSON": "{\"PIPELINE\":{\"CONFIGPATH\":\"/etc/opt/senzing\",\"LICENSESTRINGBASE64\": \"AQAAADgCAAAAAAAAU2VuemluZyBJbnRlcm5hbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAU2VuemluZyBJbnRlcm5hbAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADIwMjQtMDUtMDIAAAAAAAAAAAAARVZBTAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFNUQU5EQVJEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAC4mH8AAEBCDwAAAAAAMjAyNS0wNS0wMgAAAAAAAAAAAABZRUFSTFkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFXwDOdVd1TL+0dJRXnE9ykJJyJYnGhUN1QqoS8ASfNaDioankisRviWuB3I5uQ20EEh9tjNzzOszGf1+khWl5cb+XqE+GoMMW0rrSi6ScZmgrfh2oHrRpEbnfb4uejMrl3XGdTPdHUGNSkTKDgEQrlimVt04W5gsFVcBHBiUbKoZCghI+qaYGocsNZLh1yWOklu8Fh02CWkXXQZSKvq/PsXpkHtbsxPbActcMmZRYPZNiRXq0BK3ChyCRM0zbl4mZCPBfNL9zAx6v2HLUmDp4lNEVIyS86T9/enSrsK1udnJq09jnP8gBzY6kBxpoYyxr5o2u1VX3DC9ySHiwtio6NQMo0ckGultNqYpSBejXm10YCYH6eCsnnC5z49Gp+2NYIRcgRz/N93uLd7PrkLyLreayF8HCQOg7CBZeUGcFsufdf0304eJHCsoRy1w2dUT8N2auYJxuzjwzAMvZIYrYamjiG6Mc4Wdcpuktlcht+pjhqk9vwqQI0AzjMq2oXDGYL6KlFcOAojAIZu8bl30pZGGkq2n9NFuuO4gMiRjIwYkBpwHNmBq3QT21owPb4urlidmQelmXtzk9+BNMZL34bUK7R509Rt3GTmjb2c5TDqyIatGfnBsh3658ce8ohnBJ/ZmUgJifcorgLDawDqr8spClKfwLtcwzbkNPDKHJ/e\",\"RESOURCEPATH\":\"/opt/senzing/g2/resources\",\"SUPPORTPATH\":\"/opt/senzing/data\"},\"SQL\":{\"CONNECTION\":\"sqlite3://na:na@/tmp/sqlite/G2C.db\"}}" | ||
| }, | ||
| "python.testing.pytestArgs": [], | ||
| "python.autoComplete.extraPaths": [ | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python-core/src", | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python-core/src/senzing", | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python/src", | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python/src/senzing" | ||
| ], | ||
| "python.analysis.extraPaths": [ | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python-core/src", | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python-core/src/senzing", | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python/src", | ||
| "/home/ant/Work/Senzing/git/sz-sdk-python/src/senzing" | ||
| ], | ||
| "pylint.importStrategy": "useBundled", | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| [build-system] | ||
| requires = [ | ||
| "setuptools>=42", | ||
| "wheel" | ||
| ] | ||
| build-backend = "setuptools.build_meta" | ||
|
|
||
| [dependency-groups] | ||
| test = ["pytest", "coverage"] | ||
|
|
||
| [tool.bandit] | ||
| skips = ["B101"] | ||
|
|
||
| [tool.black] | ||
| line-length = 120 | ||
|
|
||
| [tool.flake8] | ||
| extend-ignore = ["E203", "E501", "E704", "W503"] | ||
| max-line-length = 120 | ||
|
|
||
| [tool.isort] | ||
| profile = "black" | ||
| src_paths = ["examples", "src", "tests"] | ||
|
|
||
| [[tool.mypy.overrides]] | ||
| module = "senzing_abstract.szengineflags.*" | ||
| ignore_missing_imports = "true" | ||
| warn_unused_ignores = "false" | ||
|
|
||
| [[tool.mypy.overrides]] | ||
| module = "pytest_schema.*" | ||
| ignore_missing_imports = "true" | ||
|
|
||
| [tool.pylint] | ||
| ignored-argument-names = "args|kwargs" | ||
| disable = [ | ||
| "broad-except", | ||
| "consider-using-f-string", | ||
| "line-too-long", | ||
| "missing-function-docstring", | ||
| "missing-module-docstring", | ||
| "protected-access", | ||
| "too-many-branches", | ||
| "too-many-locals", | ||
| ] | ||
| good-names = [ | ||
| "template-python" | ||
| ] | ||
| ignore = [ | ||
| "__init__.py", | ||
| "docs/source/conf.py" | ||
| ] | ||
| notes = [ | ||
| "FIXME" | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| # Python Snippets |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| # Deleting Data | ||
| The deletion snippets outline deleting previously added source records. Deleting source records removes the previously added source record from the system, completes the entity resolution process and persists outcomes in the Senzing repository. | ||
|
|
||
| Deleting a record only requires the data source code and record ID for the record to be deleted. | ||
|
|
||
| ## Snippets | ||
| * **DeleteFutures.py** | ||
| * Read and delete source records from a file using multiple threads | ||
| * **DeleteLoop.py** | ||
| * Basic read and delete source records from a file | ||
| * **DeleteWithInfoFutures.py** | ||
| * Read and delete source records from a file using multiple threads | ||
| * Collect the response from the [with info](../../../README.md#with-info) version of the API and write it to a file | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| #! /usr/bin/env python3 | ||
|
|
||
| import os | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| from senzing_core import SzAbstractFactory, SzError | ||
|
|
||
| ENGINE_CONFIG_JSON = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", "{}") | ||
| INSTANCE_NAME = Path(__file__).stem | ||
|
|
||
|
|
||
| try: | ||
| sz_factory = SzAbstractFactory("add_records", ENGINE_CONFIG_JSON, verbose_logging=False) | ||
| sz_config = sz_factory.create_config() | ||
| sz_configmanager = sz_factory.create_configmanager() | ||
|
|
||
| config_id = sz_configmanager.get_default_config_id() | ||
| config_definition = sz_configmanager.get_config(config_id) | ||
| config_handle = sz_config.import_config(config_definition) | ||
|
|
||
| for data_source in ("CUSTOMERS", "REFERENCE", "WATCHLIST"): | ||
| response = sz_config.add_data_source(config_handle, data_source) | ||
|
|
||
| config_definition = sz_config.export_config(config_handle) | ||
| config_id = sz_configmanager.add_config(config_definition, INSTANCE_NAME) | ||
| sz_configmanager.set_default_config_id(config_id) | ||
|
|
||
| response2 = sz_config.get_data_sources(config_handle) | ||
| sz_config.close_config(config_handle) | ||
| print(response2) | ||
| except SzError as err: | ||
| print(f"{err.__class__.__name__} - {err}", file=sys.stderr) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # Deleting Data | ||
|
|
||
| The deletion snippets outline deleting previously added source records. Deleting source records removes the previously added source record from the system, completes the entity resolution process and persists outcomes in the Senzing repository. | ||
|
|
||
| Deleting a record only requires the data source code and record ID for the record to be deleted. | ||
|
|
||
| ## Snippets | ||
|
|
||
| - **delete_futures.py** | ||
| - Read and delete source records from a file using multiple threads | ||
| - **delete_loop.py** | ||
| - Basic read and delete source records from a file | ||
| - **delete_with_info_futures.py** | ||
| - Read and delete source records from a file using multiple threads | ||
| - Collect the response using the [SZ_WITH_INFO flag](../../README.md#with-info) on the `delete_record()` method and write it to a file |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| #! /usr/bin/env python3 | ||
|
|
||
| import concurrent.futures | ||
| import itertools | ||
| import json | ||
| import os | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| from senzing_core import ( | ||
| SzAbstractFactory, | ||
| SzBadInputError, | ||
| SzError, | ||
| SzRetryableError, | ||
| SzUnrecoverableError, | ||
| ) | ||
|
|
||
| ENGINE_CONFIG_JSON = os.getenv("SENZING_ENGINE_CONFIGURATION_JSON", "{}") | ||
| INPUT_FILE = Path("../../resources/data/del-500.jsonl").resolve() | ||
| INSTANCE_NAME = Path(__file__).stem | ||
|
|
||
|
|
||
| def mock_logger(level, error, error_record=None): | ||
| print(f"\n{level}: {error.__class__.__name__} - {error}", file=sys.stderr) | ||
| if error_record: | ||
| print(f"{error_record}", file=sys.stderr) | ||
|
|
||
|
|
||
| def delete_record(engine, record_to_delete): | ||
| record_dict = json.loads(record_to_delete) | ||
| data_source = record_dict.get("DATA_SOURCE", "") | ||
| record_id = record_dict.get("RECORD_ID", "") | ||
| engine.delete_record(data_source, record_id) | ||
|
|
||
|
|
||
| def futures_del(engine, input_file): | ||
| success_recs = 0 | ||
| error_recs = 0 | ||
|
|
||
| with open(input_file, "r", encoding="utf-8") as in_file: | ||
| with concurrent.futures.ThreadPoolExecutor() as executor: | ||
| futures = { | ||
| executor.submit(delete_record, engine, record): record | ||
| for record in itertools.islice(in_file, executor._max_workers) | ||
| } | ||
|
|
||
| while futures: | ||
| done, _ = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED) | ||
| for f in done: | ||
| try: | ||
| f.result() | ||
| except (SzBadInputError, json.JSONDecodeError) as err: | ||
| mock_logger("ERROR", err, futures[f]) | ||
| error_recs += 1 | ||
| except SzRetryableError as err: | ||
| mock_logger("WARN", err, futures[f]) | ||
| error_recs += 1 | ||
| except (SzUnrecoverableError, SzError) as err: | ||
| mock_logger("CRITICAL", err, futures[f]) | ||
| raise err | ||
| else: | ||
| record = in_file.readline() | ||
| if record: | ||
| futures[executor.submit(delete_record, engine, record)] = record | ||
|
|
||
| success_recs += 1 | ||
| if success_recs % 100 == 0: | ||
| print(f"Processed {success_recs:,} adds, with {error_recs:,} errors", flush=True) | ||
| finally: | ||
| del futures[f] | ||
|
|
||
| print(f"\nSuccessfully deleted {success_recs:,} records, with" f" {error_recs:,} errors") | ||
|
|
||
|
|
||
| try: | ||
| sz_factory = SzAbstractFactory(INSTANCE_NAME, ENGINE_CONFIG_JSON, verbose_logging=False) | ||
| sz_engine = sz_factory.create_engine() | ||
| futures_del(sz_engine, INPUT_FILE) | ||
| except SzError as err: | ||
| mock_logger("CRITICAL", err) |
Oops, something went wrong.