From 15e2d844d42e98dca604a745d2e7444ac3f7e89f Mon Sep 17 00:00:00 2001 From: "michael@dockter.com" Date: Mon, 7 Jun 2021 20:57:47 -0400 Subject: [PATCH 1/6] #2 Update submodules for SenzingAPI 2.5.1 --- bin/05-commit-repositories.md | 4 ++-- bin/07-submodules.sh | 2 +- bin/13-create-g2-python-version.md | 2 +- g2loader | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/05-commit-repositories.md b/bin/05-commit-repositories.md index e3b5681..b767e9d 100644 --- a/bin/05-commit-repositories.md +++ b/bin/05-commit-repositories.md @@ -6,7 +6,7 @@ ```console git status -git commit -a -m "#3 2.5.0" +git commit -a -m "#3 2.5.1" git push git status ``` @@ -14,7 +14,7 @@ git status 3. Create pull request with the following title: ```console -Shipped with SenzingAPI 2.5.0 +Shipped with SenzingAPI 2.5.1 ``` 4. Pull request, but do not delete branch diff --git a/bin/07-submodules.sh b/bin/07-submodules.sh index 8f6a2cc..5641988 100755 --- a/bin/07-submodules.sh +++ b/bin/07-submodules.sh @@ -22,7 +22,7 @@ SUBMODULES=( "g2health;1.1.0;G2Health.py" "g2hasher;1.5.0;G2Hasher.py" "g2iniparams;1.1.0;G2IniParams.py" - "g2loader;1.18.0;G2Loader.py" + "g2loader;1.18.1;G2Loader.py" "g2paths;1.3.0;G2Paths.py" "g2product;1.6.0;G2Product.py" "g2project;1.11.0;G2Project.py" diff --git a/bin/13-create-g2-python-version.md b/bin/13-create-g2-python-version.md index 3acf453..3f2a848 100644 --- a/bin/13-create-g2-python-version.md +++ b/bin/13-create-g2-python-version.md @@ -5,7 +5,7 @@ In https://github.com/Senzing/g2-python: 1. Create a Pull Request with the title: ```console -SenzingAPI 2.5.0 +SenzingAPI 2.5.1 ``` 2. Create a new version that matches SenzingAPI version diff --git a/g2loader b/g2loader index 094cb50..435c853 160000 --- a/g2loader +++ b/g2loader @@ -1 +1 @@ -Subproject commit 094cb506a729d2f60b03f10be8e039e99da568ff +Subproject commit 435c85342a01b30b79f893819a9f8ce1af3ad468 From 137787c54d7d15190ded911c27aa0989f483f96d Mon Sep 17 00:00:00 2001 From: "michael@dockter.com" Date: Mon, 7 Jun 2021 20:58:08 -0400 Subject: [PATCH 2/6] #2 SenzingAPI 2.5.1 --- g2/python/G2Loader.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/g2/python/G2Loader.py b/g2/python/G2Loader.py index 33b8cf1..b236d72 100644 --- a/g2/python/G2Loader.py +++ b/g2/python/G2Loader.py @@ -458,7 +458,7 @@ def check_resources_and_startup(thread_count, doPurge, doLicense=True): if critical_error or warning_issued: print(pause_msg, flush=True) - time.sleep({10 if critical_error else 3}) + time.sleep(10 if critical_error else 3) # Purge repository if doPurge: @@ -800,9 +800,13 @@ def perform_load(): outputFile.close() # Remove shuffled file unless run with -snd or prior shuffle detected and not small file/low thread count - if not args.shuffleNoDelete and not shuf_detected and transportThreadCount > 1 and not args.noShuffle: - print(f'\nDeleting shuffled file: {shuf_file_path}') + if not args.shuffleNoDelete \ + and not shuf_detected \ + and not args.noShuffle \ + and not args.testMode \ + and transportThreadCount > 1: with suppress(Exception): + print(f'\nDeleting shuffled file: {shuf_file_path}') os.remove(shuf_file_path) # Stop processes and threads From 66185a1daf55e508a7296e3970607f55d90c772e Mon Sep 17 00:00:00 2001 From: "michael@dockter.com" Date: Mon, 7 Jun 2021 21:01:24 -0400 Subject: [PATCH 3/6] #2 Savepoint --- bin/05-commit-repositories.md | 4 ++-- bin/13-create-g2-python-version.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/05-commit-repositories.md b/bin/05-commit-repositories.md index b767e9d..fc97658 100644 --- a/bin/05-commit-repositories.md +++ b/bin/05-commit-repositories.md @@ -6,7 +6,7 @@ ```console git status -git commit -a -m "#3 2.5.1" +git commit -a -m "#3 2.6.0" git push git status ``` @@ -14,7 +14,7 @@ git status 3. Create pull request with the following title: ```console -Shipped with SenzingAPI 2.5.1 +Shipped with SenzingAPI 2.6.0 ``` 4. Pull request, but do not delete branch diff --git a/bin/13-create-g2-python-version.md b/bin/13-create-g2-python-version.md index 3f2a848..b4d1977 100644 --- a/bin/13-create-g2-python-version.md +++ b/bin/13-create-g2-python-version.md @@ -5,7 +5,7 @@ In https://github.com/Senzing/g2-python: 1. Create a Pull Request with the title: ```console -SenzingAPI 2.5.1 +SenzingAPI 2.6.0 ``` 2. Create a new version that matches SenzingAPI version From a466c1d91ca7d4d443a1cc339610fe38cc918628 Mon Sep 17 00:00:00 2001 From: "michael@dockter.com" Date: Mon, 7 Jun 2021 21:33:44 -0400 Subject: [PATCH 4/6] #2 Update submodules for SEnzingAPI 2.6.0 --- bin/07-submodules.sh | 14 +- g2/python/demo/truth/project.csv | 3 + g2/python/demo/truth/project.json | 5 + .../truth/truthset-person-v1-set1-data.csv | 2 +- .../truth/truthset-person-v1-set2-key.csv | 208 +++++++++--------- 5 files changed, 120 insertions(+), 112 deletions(-) create mode 100644 g2/python/demo/truth/project.csv create mode 100644 g2/python/demo/truth/project.json diff --git a/bin/07-submodules.sh b/bin/07-submodules.sh index 5641988..ba3b578 100755 --- a/bin/07-submodules.sh +++ b/bin/07-submodules.sh @@ -6,26 +6,26 @@ SUBMODULES=( "compressedfile;1.3.0;CompressedFile.py" "dumpstack;1.0.0;DumpStack.py" "g2audit;1.9.0;G2Audit.py" - "g2command;1.17.0;G2Command.py" + "g2command;1.18.0;G2Command.py" "g2config;1.10.0;G2Config.py" "g2configmgr;1.5.0;G2ConfigMgr.py" - "g2configtables;1.6.0;G2ConfigTables.py" - "g2configtool;1.15.0;G2ConfigTool.py" - "g2configtool;1.15.0;G2ConfigTool.readme" + "g2configtables;1.7.0;G2ConfigTables.py" + "g2configtool;1.16.0;G2ConfigTool.py" + "g2configtool;1.16.0;G2ConfigTool.readme" "g2createproject;1.8.0;G2CreateProject.py" "g2database;1.7.0;G2Database.py" "g2diagnostic;1.9.0;G2Diagnostic.py" "g2engine;1.15.0;G2Engine.py" "g2exception;1.2.0;G2Exception.py" - "g2explorer;1.4.0;G2Explorer.py" + "g2explorer;1.5.0;G2Explorer.py" "g2export;1.13.0;G2Export.py" "g2health;1.1.0;G2Health.py" "g2hasher;1.5.0;G2Hasher.py" "g2iniparams;1.1.0;G2IniParams.py" - "g2loader;1.18.1;G2Loader.py" + "g2loader;1.19.0;G2Loader.py" "g2paths;1.3.0;G2Paths.py" "g2product;1.6.0;G2Product.py" - "g2project;1.11.0;G2Project.py" + "g2project;1.12.0;G2Project.py" "g2s3;1.0.0;G2S3.py" "g2setupconfig;1.2.0;G2SetupConfig.py" "g2snapshot;1.1.0;G2Snapshot.py" diff --git a/g2/python/demo/truth/project.csv b/g2/python/demo/truth/project.csv new file mode 100644 index 0000000..deab06b --- /dev/null +++ b/g2/python/demo/truth/project.csv @@ -0,0 +1,3 @@ +DATA_SOURCE,FILE_FORMAT,FILE_NAME +CUSTOMERS,CSV,truthset-person-v1-set1-data.csv +WATCHLIST,CSV,truthset-person-v1-set2-data.csv diff --git a/g2/python/demo/truth/project.json b/g2/python/demo/truth/project.json new file mode 100644 index 0000000..0fd0eec --- /dev/null +++ b/g2/python/demo/truth/project.json @@ -0,0 +1,5 @@ +[ +{"DATA_SOURCE": "CUSTOMERS", "FILE_FORMAT": "CSV", "FILE_NAME": "truthset-person-v1-set1-data.csv"}, +{"DATA_SOURCE": "WATCHLIST", "FILE_FORMAT": "CSV", "FILE_NAME": "truthset-person-v1-set2-data.csv"} +] + diff --git a/g2/python/demo/truth/truthset-person-v1-set1-data.csv b/g2/python/demo/truth/truthset-person-v1-set1-data.csv index 5010c11..723be3c 100644 --- a/g2/python/demo/truth/truthset-person-v1-set1-data.csv +++ b/g2/python/demo/truth/truthset-person-v1-set1-data.csv @@ -1,7 +1,7 @@ RECORD_ID,PRIMARY_NAME_LAST,PRIMARY_NAME_FIRST,PRIMARY_NAME_MIDDLE,GENDER,DATE_OF_BIRTH,PASSPORT_NUMBER,PASSPORT_COUNTRY,DRIVERS_LICENSE_NUMBER,DRIVERS_LICENSE_STATE,SSN_NUMBER,SSN_LAST4,NATIONAL_ID_NUMBER,NATIONAL_ID_COUNTRY,HOME_ADDR_LINE1,HOME_ADDR_CITY,HOME_ADDR_STATE,HOME_ADDR_POSTAL_CODE,HOME_ADDR_COUNTRY,HOME_PHONE_NUMBER,CELL_PHONE_NUMBER,EMAIL_ADDRESS,DATE,STATUS,AMOUNT 1001,Smith,Robert,,,12/11/78,,,,,,,,,"123 Main Street, Las Vegas NV 89132",,,,,702-919-1300,,bsmith@work.com,1/2/2018,Active,100 1002,Smith,Bob,,,11/12/78,,,,,,,,,1515 Adela Lane,Las Vegas,NV,89111,,,702-919-1300,,3/10/2017,Inactive,200 -1003,Smith,Bob,J,,12/11/78,,,,,,,,,,,,,,,,bsmith@work.com,4/9/2016,Inactive,300 +1003,Smith,Bob,,,12/11/78,,,,,,,,,,,,,,,,bsmith@work.com,4/9/2016,Inactive,300 1004,Smith,B,,,,,,,,,,,,1515 Adela Ln,Las Vegas,NV,89132,,,,bsmith@work.com,1/5/2015,Inactive,400 1005,Smith,Rob,E,,,,,112233,NV,,,,,123 E Main St,Henderson,NV,89132,,,,,7/16/2019,Active,500 1009,Kusha,Edward,,,3/1/70,,,,,294-66-9999,,,,1304 Poppy Hills Dr,Blacklick,OH,43004,,,,Kusha123@hmail.com,1/7/2018,Active,600 diff --git a/g2/python/demo/truth/truthset-person-v1-set2-key.csv b/g2/python/demo/truth/truthset-person-v1-set2-key.csv index a12a3bc..910343f 100644 --- a/g2/python/demo/truth/truthset-person-v1-set2-key.csv +++ b/g2/python/demo/truth/truthset-person-v1-set2-key.csv @@ -1,104 +1,104 @@ -TEST_GROUP,RESOLVED_ENTITY_ID,MATCH_KEY,DATA_SOURCE,ENTITY_TYPE,RECORD_ID -TI-1000,TI-1000-1,anchor,TRUTH-SET1,PERSON,1001 -TI-1000,TI-1000-1,resolve: name var | dob swap | address | phone swap (moved),TRUTH-SET1,PERSON,1002 -TI-1000,TI-1000-1,resolve: name | dob | email,TRUTH-SET1,PERSON,1003 -TI-1000,TI-1000-1,resolve: weak name | address | email,TRUTH-SET1,PERSON,1004 -TI-1000,TI-1000-2,moved: name | address | drlic (better match to rob sr),TRUTH-SET1,PERSON,1005 -TI-1000,TI-1000-2,related: name | address | dob diff (causes un-merge),TRUTH-SET2,PERSON,1006 -TI-1000,TI-1000-3,related: sur name | address ,TRUTH-SET2,PERSON,1007 -TI-1000,TI-1000-4,norelate: name only,TRUTH-SET2,PERSON,1008 -TI-1015,TI-1015-1,anchor,TRUTH-SET1,PERSON,1009 -TI-1015,TI-1015-1,resolve: name var | dob format | address,TRUTH-SET1,PERSON,1010 -TI-1015,TI-1015-1,moved: name var | dob | address,TRUTH-SET1,PERSON,1011 -TI-1015,TI-1015-1,resolve: same name | dob (adds Jed's address),TRUTH-SET2,PERSON,1012 -TI-1015,TI-1015-1,resolve: diff name | ssn | dob | addr (causes re-resolve),TRUTH-SET2,PERSON,1014 -TI-1015,TI-1015-3,anchor,TRUTH-SET1,PERSON,1015 -TI-1015,TI-1015-3,resolve: name var | dob | addr,TRUTH-SET1,PERSON,1016 -TI-1015,TI-1015-3,resolve: name | ssn ,TRUTH-SET1,PERSON,1017 -TI-1015,TI-1015-3,resolve: name var | close dob | phone,TRUTH-SET1,PERSON,1018 -TI-1015,TI-1015-4,related: sur name | address | email,TRUTH-SET1,PERSON,1019 -TI-1015,TI-1015-5,related: sur name | address | email,TRUTH-SET1,PERSON,1020 -TI-1015,TI-1015-5,resolve: no name | ssn | address | email (better match to Marsha),TRUTH-SET1,PERSON,1021 -TI-1017,TI-1017-1,anchor,TRUTH-SET1,PERSON,1022 -TI-1017,TI-1017-1,resolve: very close name | dob,TRUTH-SET1,PERSON,1023 -TI-1017,TI-1017-1,resolve: name var | dob,TRUTH-SET2,PERSON,1024 -TI-1017,TI-1017-2,anchor,TRUTH-SET1,PERSON,1025 -TI-1017,TI-1017-3,related: close name | dob (to darla),TRUTH-SET1,PERSON,1026 -TI-1017,TI-1017-4,related: close name | dob (to darla),TRUTH-SET2,PERSON,1027 -TI-1020,TI-1020-1,anchor,TRUTH-SET1,PERSON,1028 -TI-1020,TI-1020-2,related: name | address | generation diff,TRUTH-SET2,PERSON,1029 -TI-1021,TI-1021-1,anchor,TRUTH-SET1,PERSON,1030 -TI-1021,TI-1021-1,resolve: close name | dob | address,TRUTH-SET1,PERSON,1031 -TI-1022,TI-1022-1,anchor,TRUTH-SET1,PERSON,1032 -TI-1022,TI-1022-1,resolve: name swap | dob swap | close phone,TRUTH-SET1,PERSON,1033 -TI-1023,TI-1023-1,anchor,TRUTH-SET1,PERSON,1034 -TI-1023,TI-1023-1,resolve: close name | dob | address,TRUTH-SET1,PERSON,1035 -TI-1023,TI-1023-1,resolve: close name | dob | address,TRUTH-SET1,PERSON,1036 -TI-1023,TI-1023-2,none: neither name or dob is close enough,TRUTH-SET2,PERSON,1037 -TI-1023,TI-1023-1,resolve: weak name | dob | address | email,TRUTH-SET2,PERSON,1038 -TI-1024,TI-1024-1,anchor,TRUTH-SET1,PERSON,1039 -TI-1024,TI-1024-2,relate: name | diff dob | address ,TRUTH-SET1,PERSON,1040 -TI-1024,TI-1024-3,relate: name | address (ambiguous),TRUTH-SET2,PERSON,1041 -TI-1024,TI-1024-3,relate: name | address (ambiguous),TRUTH-SET2,PERSON,1042 -TI-1025,TI-1025-1,anchor,TRUTH-SET1,PERSON,1043 -TI-1025,TI-1025-2,relate: name | diff dob | address ,TRUTH-SET1,PERSON,1044 -TI-1025,TI-1025-1,resolve: name | dob | passport (to patrick),TRUTH-SET1,PERSON,1045 -TI-1025,TI-1025-2,resolve: name | dob | passport (to patricia),TRUTH-SET1,PERSON,1046 -TI-1027,TI-1027-1,anchor,TRUTH-SET1,PERSON,1047 -TI-1027,TI-1027-1,resolve: name var | email,TRUTH-SET1,PERSON,1048 -TI-1027,TI-1027-1,resolve: name var | email,TRUTH-SET1,PERSON,1049 -TI-1028,TI-1028-1,anchor,TRUTH-SET1,PERSON,1050 -TI-1028,TI-1028-1,resolve: name var | dob | passport,TRUTH-SET1,PERSON,1051 -TI-1028,TI-1028-1,resolve: name var | passport,TRUTH-SET1,PERSON,1052 -TI-1029,TI-1029-1,anchor,TRUTH-SET1,PERSON,1053 -TI-1029,TI-1029-2,anchor,TRUTH-SET1,PERSON,1054 -TI-1029,TI-1029-1,resolve: pname | address | passport,TRUTH-SET1,PERSON,1055 -TI-1029,TI-1029-2,resolve: pname| dob | address | drlic,TRUTH-SET1,PERSON,1056 -TI-1030,TI-1030-1,anchor,TRUTH-SET1,PERSON,1057 -TI-1030,TI-1030-2,relate: name only | diff email domain,TRUTH-SET1,PERSON,1058 -TI-1031,TI-1031-1,anchor,TRUTH-SET1,PERSON,1059 -TI-1031,TI-1031-1,resolve: name | phone (initial),TRUTH-SET1,PERSON,1060 -TI-1031,TI-1031-2,anchor,TRUTH-SET1,PERSON,1061 -TI-1031,TI-1031-2,resolve: name | phone (no first name),TRUTH-SET1,PERSON,1062 -TI-1032,TI-1032-1,anchor,TRUTH-SET1,PERSON,1063 -TI-1032,TI-1032-1,resolve: name | dob (adds passport),TRUTH-SET1,PERSON,1064 -TI-1032,TI-1032-1,resolve: name | passport (adds drlic),TRUTH-SET1,PERSON,1065 -TI-1032,TI-1032-1,resolve: name | drlic (adds ssn),TRUTH-SET1,PERSON,1066 -TI-1032,TI-1032-1,resolve: name | ssn (adds address),TRUTH-SET1,PERSON,1067 -TI-1032,TI-1032-1,resolve: name | address ,TRUTH-SET1,PERSON,1068 -TI-2003,TI-2003-1,anchor,TRUTH-SET1,PERSON,1069 -TI-2003,TI-2003-1,resolve: name | gender | address | nationalID,TRUTH-SET1,PERSON,1070 -TI-2003,TI-2003-2,anchor,TRUTH-SET1,PERSON,1071 -TI-2003,TI-2003-2,resolve: name | dob | gender | address | nationalID,TRUTH-SET1,PERSON,1072 -TI-2003,TI-2003-3,anchor,TRUTH-SET1,PERSON,1073 -TI-2003,TI-2003-3,resolve: name | dob | gender | address,TRUTH-SET1,PERSON,1074 -TI-2003,TI-2003-4,anchor,TRUTH-SET1,PERSON,1075 -TI-2003,TI-2003-4,resolve: name | dob | gender | address,TRUTH-SET1,PERSON,1076 -TI-2003,TI-2003-5,anchor,TRUTH-SET1,PERSON,1077 -TI-2003,TI-2003-5,resolve: name | dob | address,TRUTH-SET1,PERSON,1078 -TI-2004,TI-2004-1,anchor,TRUTH-SET1,PERSON,1079 -TI-2004,TI-2004-1,resolve: name | dob | last4ssn,TRUTH-SET1,PERSON,1080 -TI-1034,TI-1034-1,anchor,TRUTH-SET1,PERSON,1081 -TI-1034,TI-1034-1,resolve: name | dob | address |passport,TRUTH-SET1,PERSON,1082 -TI-1034,TI-1034-2,anchor,TRUTH-SET1,PERSON,1083 -TI-1034,TI-1034-2,resolve: name | dob | address |passport,TRUTH-SET1,PERSON,1084 -TI-1034,TI-1034-3,anchor,TRUTH-SET1,PERSON,1085 -TI-1034,TI-1034-3,resolve: name | dob | phone |email,TRUTH-SET1,PERSON,1086 -TI-1035,TI-1035-1,anchor,TRUTH-SET1,PERSON,1087 -TI-1035,TI-1035-1,resolve: name | dob | phone | email | gender diff,TRUTH-SET1,PERSON,1088 -TI-1035,TI-1035-3,anchor,TRUTH-SET1,PERSON,1089 -TI-1035,TI-1035-4,resolve: name | dob | address | passport / diff generation,TRUTH-SET1,PERSON,1090 -TI-1035,TI-1035-5,anchor,TRUTH-SET1,PERSON,1091 -TI-1035,TI-1035-5,resolve: name | dob | phone |email,TRUTH-SET1,PERSON,1092 -TI-1036,TI-1036-1,anchor,TRUTH-SET1,PERSON,1093 -TI-1036,TI-1036-1,resolve: name | dob | drlic ,TRUTH-SET1,PERSON,1094 -TI-1036,TI-1036-3,anchor,TRUTH-SET1,PERSON,1095 -TI-1036,TI-1036-3,resolve: name | dob | drlic | invalid gender,TRUTH-SET1,PERSON,1096 -TI-1037,TI-1037-1,anchor,TRUTH-SET1,PERSON,1097 -TI-1037,TI-1037-1,resolve: name | address | email ,TRUTH-SET1,PERSON,1098 -TI-1037,TI-1037-3,anchor,TRUTH-SET1,PERSON,1099 -TI-1037,TI-1037-3,resolve: name | address | email | invalid gender,TRUTH-SET1,PERSON,1100 -TI-1038,TI-1038-1,anchor,TRUTH-SET1,PERSON,1101 -TI-1038,TI-1038-1,resolve: name | email ,TRUTH-SET1,PERSON,1102 -TI-1038,TI-1038-3,anchor,TRUTH-SET1,PERSON,1103 -TI-1038,TI-1038-3,resolve: name | address | email | invalid gender,TRUTH-SET1,PERSON,1104 \ No newline at end of file +TEST_GROUP,RESOLVED_ENTITY_ID,MATCH_KEY,DATA_SOURCE,ENTITY_TYPE,RECORD_ID +TI-1000,TI-1000-1,anchor,TRUTH-SET1,PERSON,1001 +TI-1000,TI-1000-1,resolve: name var | dob swap | address | phone swap (moved),TRUTH-SET1,PERSON,1002 +TI-1000,TI-1000-1,resolve: name | dob | email,TRUTH-SET1,PERSON,1003 +TI-1000,TI-1000-1,resolve: weak name | address | email,TRUTH-SET1,PERSON,1004 +TI-1000,TI-1000-2,moved: name | address | drlic (better match to rob sr),TRUTH-SET1,PERSON,1005 +TI-1000,TI-1000-2,related: name | address | dob diff (causes un-merge),TRUTH-SET2,PERSON,1006 +TI-1000,TI-1000-3,related: sur name | address ,TRUTH-SET2,PERSON,1007 +TI-1000,TI-1000-4,norelate: name only,TRUTH-SET2,PERSON,1008 +TI-1015,TI-1015-1,anchor,TRUTH-SET1,PERSON,1009 +TI-1015,TI-1015-1,resolve: name var | dob format | address,TRUTH-SET1,PERSON,1010 +TI-1015,TI-1015-1,moved: name var | dob | address,TRUTH-SET1,PERSON,1011 +TI-1015,TI-1015-1,resolve: same name | dob (adds Jed's address),TRUTH-SET2,PERSON,1012 +TI-1015,TI-1015-1,resolve: diff name | ssn | dob | addr (causes re-resolve),TRUTH-SET2,PERSON,1014 +TI-1015,TI-1015-3,anchor,TRUTH-SET1,PERSON,1015 +TI-1015,TI-1015-3,resolve: name var | dob | addr,TRUTH-SET1,PERSON,1016 +TI-1015,TI-1015-3,resolve: name | ssn ,TRUTH-SET1,PERSON,1017 +TI-1015,TI-1015-3,resolve: name var | close dob | phone,TRUTH-SET1,PERSON,1018 +TI-1015,TI-1015-4,related: sur name | address | email,TRUTH-SET1,PERSON,1019 +TI-1015,TI-1015-5,related: sur name | address | email,TRUTH-SET1,PERSON,1020 +TI-1015,TI-1015-5,resolve: no name | ssn | address | email (better match to Marsha),TRUTH-SET2,PERSON,1021 +TI-1017,TI-1017-1,anchor,TRUTH-SET1,PERSON,1022 +TI-1017,TI-1017-1,resolve: very close name | dob,TRUTH-SET1,PERSON,1023 +TI-1017,TI-1017-1,resolve: name var | dob,TRUTH-SET2,PERSON,1024 +TI-1017,TI-1017-2,anchor,TRUTH-SET1,PERSON,1025 +TI-1017,TI-1017-3,related: close name | dob (to darla),TRUTH-SET1,PERSON,1026 +TI-1017,TI-1017-4,related: close name | dob (to darla),TRUTH-SET2,PERSON,1027 +TI-1020,TI-1020-1,anchor,TRUTH-SET1,PERSON,1028 +TI-1020,TI-1020-2,related: name | address | generation diff,TRUTH-SET2,PERSON,1029 +TI-1021,TI-1021-1,anchor,TRUTH-SET1,PERSON,1030 +TI-1021,TI-1021-1,resolve: close name | dob | address,TRUTH-SET1,PERSON,1031 +TI-1022,TI-1022-1,anchor,TRUTH-SET1,PERSON,1032 +TI-1022,TI-1022-1,resolve: name swap | dob swap | close phone,TRUTH-SET1,PERSON,1033 +TI-1023,TI-1023-1,anchor,TRUTH-SET1,PERSON,1034 +TI-1023,TI-1023-1,resolve: close name | dob | address,TRUTH-SET1,PERSON,1035 +TI-1023,TI-1023-1,resolve: close name | dob | address,TRUTH-SET1,PERSON,1036 +TI-1023,TI-1023-2,none: neither name or dob is close enough,TRUTH-SET2,PERSON,1037 +TI-1023,TI-1023-1,resolve: weak name | dob | address | email,TRUTH-SET2,PERSON,1038 +TI-1024,TI-1024-1,anchor,TRUTH-SET1,PERSON,1039 +TI-1024,TI-1024-2,relate: name | diff dob | address ,TRUTH-SET1,PERSON,1040 +TI-1024,TI-1024-3,relate: name | address (ambiguous),TRUTH-SET2,PERSON,1041 +TI-1024,TI-1024-3,relate: name | address (ambiguous),TRUTH-SET2,PERSON,1042 +TI-1025,TI-1025-1,anchor,TRUTH-SET1,PERSON,1043 +TI-1025,TI-1025-2,relate: name | diff dob | address ,TRUTH-SET1,PERSON,1044 +TI-1025,TI-1025-1,resolve: name | dob | passport (to patrick),TRUTH-SET1,PERSON,1045 +TI-1025,TI-1025-2,resolve: name | dob | passport (to patricia),TRUTH-SET1,PERSON,1046 +TI-1027,TI-1027-1,anchor,TRUTH-SET1,PERSON,1047 +TI-1027,TI-1027-1,resolve: name var | email,TRUTH-SET1,PERSON,1048 +TI-1027,TI-1027-1,resolve: name var | email,TRUTH-SET1,PERSON,1049 +TI-1028,TI-1028-1,anchor,TRUTH-SET1,PERSON,1050 +TI-1028,TI-1028-1,resolve: name var | dob | passport,TRUTH-SET1,PERSON,1051 +TI-1028,TI-1028-1,resolve: name var | passport,TRUTH-SET1,PERSON,1052 +TI-1029,TI-1029-1,anchor,TRUTH-SET1,PERSON,1053 +TI-1029,TI-1029-2,anchor,TRUTH-SET1,PERSON,1054 +TI-1029,TI-1029-1,resolve: pname | address | passport,TRUTH-SET1,PERSON,1055 +TI-1029,TI-1029-2,resolve: pname| dob | address | drlic,TRUTH-SET1,PERSON,1056 +TI-1030,TI-1030-1,anchor,TRUTH-SET1,PERSON,1057 +TI-1030,TI-1030-2,relate: name only | diff email domain,TRUTH-SET1,PERSON,1058 +TI-1031,TI-1031-1,anchor,TRUTH-SET1,PERSON,1059 +TI-1031,TI-1031-1,resolve: name | phone (initial),TRUTH-SET1,PERSON,1060 +TI-1031,TI-1031-2,anchor,TRUTH-SET1,PERSON,1061 +TI-1031,TI-1031-2,resolve: name | phone (no first name),TRUTH-SET1,PERSON,1062 +TI-1032,TI-1032-1,anchor,TRUTH-SET1,PERSON,1063 +TI-1032,TI-1032-1,resolve: name | dob (adds passport),TRUTH-SET1,PERSON,1064 +TI-1032,TI-1032-1,resolve: name | passport (adds drlic),TRUTH-SET1,PERSON,1065 +TI-1032,TI-1032-1,resolve: name | drlic (adds ssn),TRUTH-SET1,PERSON,1066 +TI-1032,TI-1032-1,resolve: name | ssn (adds address),TRUTH-SET1,PERSON,1067 +TI-1032,TI-1032-1,resolve: name | address ,TRUTH-SET1,PERSON,1068 +TI-2003,TI-2003-1,anchor,TRUTH-SET1,PERSON,1069 +TI-2003,TI-2003-1,resolve: name | gender | address | nationalID,TRUTH-SET1,PERSON,1070 +TI-2003,TI-2003-2,anchor,TRUTH-SET1,PERSON,1071 +TI-2003,TI-2003-2,resolve: name | dob | gender | address | nationalID,TRUTH-SET1,PERSON,1072 +TI-2003,TI-2003-3,anchor,TRUTH-SET1,PERSON,1073 +TI-2003,TI-2003-3,resolve: name | dob | gender | address,TRUTH-SET1,PERSON,1074 +TI-2003,TI-2003-4,anchor,TRUTH-SET1,PERSON,1075 +TI-2003,TI-2003-4,resolve: name | dob | gender | address,TRUTH-SET1,PERSON,1076 +TI-2003,TI-2003-5,anchor,TRUTH-SET1,PERSON,1077 +TI-2003,TI-2003-5,resolve: name | dob | address,TRUTH-SET1,PERSON,1078 +TI-2004,TI-2004-1,anchor,TRUTH-SET1,PERSON,1079 +TI-2004,TI-2004-1,resolve: name | dob | last4ssn,TRUTH-SET1,PERSON,1080 +TI-1034,TI-1034-1,anchor,TRUTH-SET1,PERSON,1081 +TI-1034,TI-1034-1,resolve: name | dob | address |passport,TRUTH-SET1,PERSON,1082 +TI-1034,TI-1034-2,anchor,TRUTH-SET1,PERSON,1083 +TI-1034,TI-1034-2,resolve: name | dob | address |passport,TRUTH-SET1,PERSON,1084 +TI-1034,TI-1034-3,anchor,TRUTH-SET1,PERSON,1085 +TI-1034,TI-1034-3,resolve: name | dob | phone |email,TRUTH-SET1,PERSON,1086 +TI-1035,TI-1035-1,anchor,TRUTH-SET1,PERSON,1087 +TI-1035,TI-1035-1,resolve: name | dob | phone | email | gender diff,TRUTH-SET1,PERSON,1088 +TI-1035,TI-1035-3,anchor,TRUTH-SET1,PERSON,1089 +TI-1035,TI-1035-4,resolve: name | dob | address | passport / diff generation,TRUTH-SET1,PERSON,1090 +TI-1035,TI-1035-5,anchor,TRUTH-SET1,PERSON,1091 +TI-1035,TI-1035-5,resolve: name | dob | phone |email,TRUTH-SET1,PERSON,1092 +TI-1036,TI-1036-1,anchor,TRUTH-SET1,PERSON,1093 +TI-1036,TI-1036-1,resolve: name | dob | drlic ,TRUTH-SET1,PERSON,1094 +TI-1036,TI-1036-3,anchor,TRUTH-SET1,PERSON,1095 +TI-1036,TI-1036-3,resolve: name | dob | drlic | invalid gender,TRUTH-SET1,PERSON,1096 +TI-1037,TI-1037-1,anchor,TRUTH-SET1,PERSON,1097 +TI-1037,TI-1037-1,resolve: name | address | email ,TRUTH-SET1,PERSON,1098 +TI-1037,TI-1037-3,anchor,TRUTH-SET1,PERSON,1099 +TI-1037,TI-1037-3,resolve: name | address | email | invalid gender,TRUTH-SET1,PERSON,1100 +TI-1038,TI-1038-1,anchor,TRUTH-SET1,PERSON,1101 +TI-1038,TI-1038-1,resolve: name | email ,TRUTH-SET1,PERSON,1102 +TI-1038,TI-1038-3,anchor,TRUTH-SET1,PERSON,1103 +TI-1038,TI-1038-3,resolve: name | address | email | invalid gender,TRUTH-SET1,PERSON,1104 From 0182149564858395c3bae1afc062254bbdd0a1d1 Mon Sep 17 00:00:00 2001 From: "michael@dockter.com" Date: Mon, 7 Jun 2021 21:34:38 -0400 Subject: [PATCH 5/6] #2 Update submodules for SenzingAPI 2.6.0 --- g2command | 2 +- g2configtables | 2 +- g2configtool | 2 +- g2explorer | 2 +- g2loader | 2 +- g2project | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/g2command b/g2command index ea806bb..094f8f9 160000 --- a/g2command +++ b/g2command @@ -1 +1 @@ -Subproject commit ea806bba0536cf7f22d1b9e5ea6fc90c43de6ab5 +Subproject commit 094f8f910b32c6a1afc8d179ea1e6fe8384b2daf diff --git a/g2configtables b/g2configtables index c9d44b6..ce0cc2c 160000 --- a/g2configtables +++ b/g2configtables @@ -1 +1 @@ -Subproject commit c9d44b644de63692b3f0b1211bdbd1e492f60506 +Subproject commit ce0cc2c1edc43d16f59dbbcfe86e70a265648200 diff --git a/g2configtool b/g2configtool index 75b8a77..d28536e 160000 --- a/g2configtool +++ b/g2configtool @@ -1 +1 @@ -Subproject commit 75b8a770758e1caf99aa870ecf4ac73c6072fed5 +Subproject commit d28536e6194b1ea315f3a36696d79ee477f59605 diff --git a/g2explorer b/g2explorer index 006bd2f..9e83354 160000 --- a/g2explorer +++ b/g2explorer @@ -1 +1 @@ -Subproject commit 006bd2fb7d9268c16b27496e01425d57d9920eb5 +Subproject commit 9e83354673c5777c7a7652d5958b9ee92d7dc73d diff --git a/g2loader b/g2loader index 435c853..ba0ee6e 160000 --- a/g2loader +++ b/g2loader @@ -1 +1 @@ -Subproject commit 435c85342a01b30b79f893819a9f8ce1af3ad468 +Subproject commit ba0ee6e4313fbc2b346c3214e995faa43bf32701 diff --git a/g2project b/g2project index 3197254..450d39b 160000 --- a/g2project +++ b/g2project @@ -1 +1 @@ -Subproject commit 31972540c1311f3ce95d2c76b53c4b7607b26a95 +Subproject commit 450d39b647ac7866d0069f65a5171c4551b78d4c From b72bd0971974fab952d6e59f0c1c1e7bdb8916ec Mon Sep 17 00:00:00 2001 From: "michael@dockter.com" Date: Mon, 7 Jun 2021 21:35:28 -0400 Subject: [PATCH 6/6] #2 SenzingAPI 2.6.0 --- g2/python/G2Command.py | 645 ++++++++--------- g2/python/G2ConfigTables.py | 53 +- g2/python/G2ConfigTool.py | 1354 +++++++++++++++-------------------- g2/python/G2Explorer.py | 306 ++++---- g2/python/G2Loader.py | 291 ++++---- g2/python/G2Project.py | 703 +++++++++--------- 6 files changed, 1525 insertions(+), 1827 deletions(-) diff --git a/g2/python/G2Command.py b/g2/python/G2Command.py index 2c7bb71..0bf3a62 100644 --- a/g2/python/G2Command.py +++ b/g2/python/G2Command.py @@ -11,6 +11,7 @@ import sys import textwrap from collections import OrderedDict +from contextlib import suppress from timeit import default_timer as timer import G2Exception @@ -33,12 +34,12 @@ class G2CmdShell(cmd.Cmd, object): - #Override function from cmd module to make command completion case insensitive + #Override function from cmd module to make command completion case insensitive def completenames(self, text, *ignored): - dotext = 'do_'+text - return [a[3:] for a in self.get_names() if a.lower().startswith(dotext.lower())] + dotext = 'do_' + text + return [a[3:] for a in self.get_names() if a.lower().startswith(dotext.lower())] - def __init__(self, debug_trace, hist_disable, ini_file = None): + def __init__(self, debug_trace, hist_disable, ini_file=None): cmd.Cmd.__init__(self) @@ -47,7 +48,7 @@ def __init__(self, debug_trace, hist_disable, ini_file = None): self.prompt = '(g2cmd) ' self.ruler = '-' self.doc_header = 'Senzing APIs' - self.misc_header = 'Help Topics (help )' + self.misc_header = 'Help Topics (help )' self.undoc_header = 'Misc Commands' self.__hidden_methods = ('do_shell', 'do_EOF', 'do_help') @@ -65,18 +66,18 @@ def __init__(self, debug_trace, hist_disable, ini_file = None): self.quit = False self.timerOn = False self.timerStart = self.timerEnd = None - - # Readline and history + + # Readline and history self.readlineAvail = True if 'readline' in sys.modules else False self.histDisable = hist_disable self.histCheck() - self.parser = argparse.ArgumentParser(prog='', add_help=False) + self.parser = argparse.ArgumentParser(prog='', add_help=False) self.subparsers = self.parser.add_subparsers() jsonOnly_parser = self.subparsers.add_parser('jsonOnly', usage=argparse.SUPPRESS) jsonOnly_parser.add_argument('jsonData') - + jsonWithInfo_parser = self.subparsers.add_parser('jsonWithInfo', usage=argparse.SUPPRESS) jsonWithInfo_parser.add_argument('jsonData') jsonWithInfo_parser.add_argument('-f', '--flags', required=False, type=int) @@ -95,7 +96,7 @@ def __init__(self, debug_trace, hist_disable, ini_file = None): replaceDefaultConfigID_parser.add_argument('oldConfigID', type=int) replaceDefaultConfigID_parser.add_argument('newConfigID', type=int) - interfaceName_parser = self.subparsers.add_parser('interfaceName', usage=argparse.SUPPRESS) + interfaceName_parser = self.subparsers.add_parser('interfaceName', usage=argparse.SUPPRESS) interfaceName_parser.add_argument('interfaceName') searchByAttributesV2_parser = self.subparsers.add_parser('searchByAttributesV2', usage=argparse.SUPPRESS) @@ -256,7 +257,7 @@ def __init__(self, debug_trace, hist_disable, ini_file = None): reevaluateRecord_parser.add_argument('dataSourceCode') reevaluateRecord_parser.add_argument('recordID') reevaluateRecord_parser.add_argument('flags', type=int) - + reevaluateRecordWithInfo_parser = self.subparsers.add_parser('reevaluateRecordWithInfo', usage=argparse.SUPPRESS) reevaluateRecordWithInfo_parser.add_argument('dataSourceCode') reevaluateRecordWithInfo_parser.add_argument('recordID') @@ -265,7 +266,7 @@ def __init__(self, debug_trace, hist_disable, ini_file = None): reevaluateEntity_parser = self.subparsers.add_parser('reevaluateEntity', usage=argparse.SUPPRESS) reevaluateEntity_parser.add_argument('entityID', type=int) reevaluateEntity_parser.add_argument('flags', type=int) - + reevaluateEntityWithInfo_parser = self.subparsers.add_parser('reevaluateEntityWithInfo', usage=argparse.SUPPRESS) reevaluateEntityWithInfo_parser.add_argument('entityID', type=int) reevaluateEntityWithInfo_parser.add_argument('-f', '--flags', required=False, type=int) @@ -375,11 +376,11 @@ def __init__(self, debug_trace, hist_disable, ini_file = None): def preloop(self): - + if (self.initialized): return - printWithNewLines('Initializing Senzing engine...', 'S') + printWithNewLines('Initializing Senzing engines...', 'S') try: self.g2_module.initV2('pyG2E', g2module_params, self.debug_trace) @@ -388,11 +389,13 @@ def preloop(self): self.g2_config_module.initV2('pyG2Config', g2module_params, self.debug_trace) self.g2_configmgr_module.initV2('pyG2ConfigMgr', g2module_params, self.debug_trace) except G2Exception.G2Exception as ex: - printWithNewLines(f'G2Exception: {ex}', 'B') - raise + printWithNewLines(f'ERROR: {ex}', 'B') + # Clean up before exiting + self.postloop() + sys.exit(1) - exportedConfig = bytearray() - exportedConfigID = bytearray() + exportedConfig = bytearray() + exportedConfigID = bytearray() self.g2_module.exportConfig(exportedConfig, exportedConfigID) self.g2_hasher_module.initWithConfigV2('pyG2Hasher', g2module_params, exportedConfig, self.debug_trace) @@ -402,31 +405,31 @@ def preloop(self): def postloop(self): - if (self.initialized): - self.g2_module.destroy() - self.g2_product_module.destroy() - self.g2_diagnostic_module.destroy() - self.g2_config_module.destroy() - self.g2_configmgr_module.destroy() + with suppress(Exception): self.g2_hasher_module.destroy() + self.g2_configmgr_module.destroy() + self.g2_config_module.destroy() + self.g2_diagnostic_module.destroy() + self.g2_product_module.destroy() + self.g2_module.destroy() self.initialized = False def precmd(self, line): - - if self.timerOn: + + if self.timerOn: #Reset timer for every command self.timerStart = self.timerEnd = None - self.timerStart = timer() + self.timerStart = timer() return cmd.Cmd.precmd(self, line) - + def postcmd(self, stop, line): - + if self.timerOn and self.timerStart: - self.timerEnd = timer() + self.timerEnd = timer() self.execTime = (self.timerEnd - self.timerStart) printWithNewLines(f'Approximate execution time (s): {self.execTime:.5f}\n', 'N') @@ -443,7 +446,6 @@ def do_exit(self, arg): return True - def ret_quit(self): return self.quit @@ -467,18 +469,16 @@ def cmdloop(self, intro=None): while True: try: - super(G2CmdShell,self).cmdloop(intro=self.intro) - self.postloop() + super(G2CmdShell, self).cmdloop(intro=self.intro) break except KeyboardInterrupt: - if input('\n\nAre you sure you want to exit? (y/n) ') in ['y','Y', 'yes', 'YES']: + if input('\n\nAre you sure you want to exit? (y/n) ') in ['y', 'Y', 'yes', 'YES']: break else: - print() + print() except TypeError as ex: printWithNewLines(f'ERROR: {ex}', 'B') - def fileloop(self, fileName): self.preloop() @@ -501,7 +501,7 @@ def fileloop(self, fileName): try: exec_cmd = f'self.{process_cmd}({repr(" ".join(args))})' exec(exec_cmd) - except (ValueError, TypeError) as ex: + except (ValueError, TypeError) as ex: printWithNewLines('ERROR: Command could not be run!', 'S') printWithNewLines(f' {ex}', 'E') @@ -517,10 +517,10 @@ def help_Arguments(self): printWithNewLines(textwrap.dedent('''\ - Optional arguments are surrounded with [ ] - - Argument values to specify are surrounded with < >, replace with your value - + - Argument values to specify are surrounded with < >, replace with your value + - Example: - + [-o ] - -o = an optional argument @@ -553,9 +553,9 @@ def help_History(self): printWithNewLines(textwrap.dedent(f'''\ - Use shell like history, requires Python readline module. - - Tries to create a history file in the users home directory for use across instances of G2Command. + - Tries to create a history file in the users home directory for use across instances of G2Command. - - If a history file can't be created in the users home, /tmp is tried for temporary session history. + - If a history file can't be created in the users home, /tmp is tried for temporary session history. - Ctrl-r can be used to search history when history is available @@ -565,7 +565,7 @@ def help_History(self): - histDedupe = The history can accumulate duplicate entries over time, use this to remove them - histShow = Display all history - - History Status: + - History Status: - Readline available: {self.readlineAvail} - History available: {self.histAvail} - History file: {self.histFileName} @@ -590,42 +590,42 @@ def do_shell(self,line): def histCheck(self): ''' ''' - + self.histFileName = None self.histFileError = None self.histAvail = False - + if not self.histDisable: - + if readline: - tmpHist = '.' + os.path.basename(sys.argv[0].lower().replace('.py','_history')) + tmpHist = '.' + os.path.basename(sys.argv[0].lower().replace('.py', '_history')) self.histFileName = os.path.join(os.path.expanduser('~'), tmpHist) - - #Try and open history in users home first for longevity + + #Try and open history in users home first for longevity try: open(self.histFileName, 'a').close() except IOError as e: self.histFileError = f'{e} - Couldn\'t use home, trying /tmp/...' - + #Can't use users home, try using /tmp/ for history useful at least in the session if self.histFileError: - + self.histFileName = f'/tmp/{tmpHist}' try: open(self.histFileName, 'a').close() except IOError as e: self.histFileError = f'{e} - User home dir and /tmp/ failed!' return - + hist_size = 2000 readline.read_history_file(self.histFileName) readline.set_history_length(hist_size) atexit.register(readline.set_history_length, hist_size) atexit.register(readline.write_history_file, self.histFileName) - + self.histFileName = self.histFileName self.histFileError = None - self.histAvail = True + self.histAvail = True # ----- exception commands ----- @@ -638,7 +638,7 @@ def do_clearLastException(self,arg): print(self.do_clearLastException.__doc__) return - try: + try: if args.interfaceName == 'engine': self.g2_module.clearLastException() elif args.interfaceName == 'hasher': @@ -666,7 +666,7 @@ def do_getLastException(self,arg): print(self.do_getLastException.__doc__) return - try: + try: resultString = '' if args.interfaceName == 'engine': resultString = self.g2_module.getLastException() @@ -694,8 +694,8 @@ def do_getLastExceptionCode(self,arg): except SystemExit: print(self.do_getLastExceptionCode.__doc__) return - - try: + + try: resultInt = 0 if args.interfaceName == 'engine': resultInt = self.g2_module.getLastExceptionCode() @@ -722,7 +722,7 @@ def do_getLastExceptionCode(self,arg): def do_primeEngine(self,arg): '\nPrime the G2 engine: primeEngine\n' - try: + try: self.g2_module.primeEngine() except G2Exception.G2Exception as err: print(err) @@ -737,7 +737,7 @@ def do_process(self, arg): print(self.do_process.__doc__) return - try: + try: self.g2_module.process(args.jsonData) print() except G2Exception.G2Exception as err: @@ -758,8 +758,8 @@ def do_processWithInfo(self, arg): if args.flags: flags = int(args.flags) - response = bytearray() - self.g2_module.processWithInfo(args.jsonData,response,flags=flags) + response = bytearray() + self.g2_module.processWithInfo(args.jsonData, response, flags=flags) if response: print('{}'.format(response.decode())) else: @@ -777,7 +777,7 @@ def do_processFile(self, arg): print(self.do_processFile.__doc__) return - try: + try: dataSourceParm = None if '/?' in args.inputFile: fileName, dataSourceParm = args.inputFile.split("/?") @@ -788,7 +788,7 @@ def do_processFile(self, arg): fileName = args.inputFile dummy, fileExtension = os.path.splitext(fileName) fileExtension = fileExtension[1:].upper() - + with open(fileName) as data_in: if fileExtension != 'CSV': fileReader = data_in @@ -805,17 +805,17 @@ def do_processFile(self, arg): jsonObj['DATA_SOURCE'] = dataSourceParm jsonObj['ENTITY_TYPE'] = dataSourceParm jsonStr = json.dumps(jsonObj) - + self.g2_module.process(jsonStr) cnt += 1 if cnt % 1000 ==0: print('%s rows processed' % cnt) print('%s rows processed, done!' % cnt) - + print() except G2Exception.G2Exception as err: print(err) - + def do_processWithResponse(self, arg): '\nProcess a generic record, and print response: processWithResponse [-o ]\n' @@ -835,9 +835,9 @@ def do_processWithResponse(self, arg): data_out.write('\n') print() else: - processedData = bytearray() - self.g2_module.processWithResponse(args.jsonData,processedData) - printResponse(processedData.decode()) + processedData = bytearray() + self.g2_module.processWithResponse(args.jsonData,processedData) + printResponse(processedData.decode()) except G2Exception.G2Exception as err: print(err) @@ -851,7 +851,7 @@ def do_processFileWithResponse(self, arg): print(self.do_processFileWithResponse.__doc__) return - try: + try: if args.outputFile: with open(args.outputFile, 'w') as data_out: with open(args.inputFile.split("?")[0]) as data_in: @@ -872,7 +872,7 @@ def do_processFileWithResponse(self, arg): def do_exportCSVEntityReportV2(self, arg): - '\nExport repository contents as CSV: exportCSVEntityReportV2 -t -f [-o ]\n' + '\nExport repository contents as CSV: exportCSVEntityReportV2 -t -f [-o ]\n' try: args = self.parser.parse_args(['exportEntityCsvV2'] + parse(arg)) @@ -880,9 +880,9 @@ def do_exportCSVEntityReportV2(self, arg): print(self.do_exportCSVEntityReportV2.__doc__) return - try: + try: exportHandle = self.g2_module.exportCSVEntityReportV2(args.headersForCSV, args.flags) - response = bytearray() + response = bytearray() rowData = self.g2_module.fetchNext(exportHandle,response) recCnt = 0 resultString = b"" @@ -905,7 +905,7 @@ def do_exportCSVEntityReportV2(self, arg): def do_exportJSONEntityReport(self, arg): - '\nExport repository contents as JSON: exportJSONEntityReport -f [-o ]\n' + '\nExport repository contents as JSON: exportJSONEntityReport -f [-o ]\n' try: args = self.parser.parse_args(['exportEntityReport'] + parse(arg)) @@ -913,9 +913,9 @@ def do_exportJSONEntityReport(self, arg): print(self.do_exportJSONEntityReport.__doc__) return - try: + try: exportHandle = self.g2_module.exportJSONEntityReport(args.flags) - response = bytearray() + response = bytearray() rowData = self.g2_module.fetchNext(exportHandle,response) recCnt = 0 resultString = b"" @@ -935,17 +935,16 @@ def do_exportJSONEntityReport(self, arg): else: print('Number of exported records = %s\n' % (recCnt) ) - def do_exportCSVEntityReportV3(self, arg): - '\nExport repository contents as CSV: exportCSVEntityReportV3 -t -f [-o ]\n' + '\nExport repository contents as CSV: exportCSVEntityReportV3 -t -f [-o ]\n' try: args = self.parser.parse_args(['exportEntityCsvV3'] + parse(arg)) except SystemExit: print(self.do_exportCSVEntityReportV3.__doc__) return - try: + try: exportHandle = self.g2_module.exportCSVEntityReportV3(args.headersForCSV, args.flags) - response = bytearray() + response = bytearray() rowData = self.g2_module.fetchNextV3(exportHandle,response) recCnt = 0 resultString = b"" @@ -963,17 +962,18 @@ def do_exportCSVEntityReportV3(self, arg): except G2Exception.G2Exception as err: print(err) else: - print('Number of exported records = %s\n' % (recCnt-1) ) + print('Number of exported records = %s\n' % (recCnt - 1)) + def do_exportJSONEntityReportV3(self, arg): - '\nExport repository contents as JSON: exportJSONEntityReportV3 -f [-o ]\n' + '\nExport repository contents as JSON: exportJSONEntityReportV3 -f [-o ]\n' try: args = self.parser.parse_args(['exportEntityReport'] + parse(arg)) except SystemExit: print(self.do_exportJSONEntityReportV3.__doc__) return - try: + try: exportHandle = self.g2_module.exportJSONEntityReportV3(args.flags) - response = bytearray() + response = bytearray() rowData = self.g2_module.fetchNextV3(exportHandle,response) recCnt = 0 resultString = b"" @@ -981,7 +981,7 @@ def do_exportJSONEntityReportV3(self, arg): resultString += response recCnt = recCnt + 1 response = bytearray() - rowData = self.g2_module.fetchNextV3(exportHandle,response) + rowData = self.g2_module.fetchNextV3(exportHandle, response) self.g2_module.closeExportV3(exportHandle) if args.outputFile: with open(args.outputFile, 'w') as data_out: @@ -991,14 +991,16 @@ def do_exportJSONEntityReportV3(self, arg): except G2Exception.G2Exception as err: print(err) else: - print('Number of exported records = %s\n' % (recCnt) ) + print('Number of exported records = %s\n' % (recCnt)) + + def do_getTemplateConfig(self, arg): '\nGet a template config: getTemplateConfig \n' try: configHandle = self.g2_config_module.create() - response = bytearray() - self.g2_config_module.save(configHandle,response) + response = bytearray() + self.g2_config_module.save(configHandle, response) self.g2_config_module.close(configHandle) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -1015,8 +1017,8 @@ def do_getConfig(self, arg): return try: - response = bytearray() - self.g2_configmgr_module.getConfig(args.configID,response) + response = bytearray() + self.g2_configmgr_module.getConfig(args.configID, response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: print(err) @@ -1026,7 +1028,7 @@ def do_getConfigList(self, arg): '\nGet a list of known configs: getConfigList \n' try: - response = bytearray() + response = bytearray() self.g2_configmgr_module.getConfigList(response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -1047,8 +1049,8 @@ def do_addConfigFile(self, arg): with open(args.configJsonFile.split("?")[0]) as data_in: for line in data_in: configStr += line.strip() - configID = bytearray() - self.g2_configmgr_module.addConfig(configStr,args.configComments,configID) + configID = bytearray() + self.g2_configmgr_module.addConfig(configStr, args.configComments, configID) printWithNewLine('Config added. [ID = %s]' % configID.decode()) except G2Exception.G2Exception as err: print(err) @@ -1058,13 +1060,12 @@ def do_getDefaultConfigID(self, arg): '\nGet the default config ID: getDefaultConfigID \n' try: - configID = bytearray() + configID = bytearray() self.g2_configmgr_module.getDefaultConfigID(configID) printWithNewLine('Default Config ID: \'%s\'' % (configID.decode())) except G2Exception.G2Exception as err: print(err) - def do_setDefaultConfigID(self, arg): '\nSet the default config ID: setDefaultConfigID \n' @@ -1076,11 +1077,12 @@ def do_setDefaultConfigID(self, arg): try: self.g2_configmgr_module.setDefaultConfigID(str(args.configID).encode()) - printWithNewLine('Default config set') + printWithNewLines('Default config set, restarting engines...', 'B') + self.do_restart(None) if not self.debug_trace else self.do_restartDebug(None) + return True except G2Exception.G2Exception as err: print(err) - def do_replaceDefaultConfigID(self, arg): '\nReplace the default config ID: replaceDefaultConfigID \n' @@ -1091,8 +1093,11 @@ def do_replaceDefaultConfigID(self, arg): return try: - self.g2_configmgr_module.replaceDefaultConfigID(args.oldConfigID,args.newConfigID) - printWithNewLine('New default config set') + self.g2_configmgr_module.replaceDefaultConfigID(args.oldConfigID, args.newConfigID) + printWithNewLines('New default config set, restarting engines...', 'B') + self.do_restart(None) if not self.debug_trace else self.do_restartDebug(None) + return True + except G2Exception.G2Exception as err: print(err) @@ -1106,7 +1111,7 @@ def do_addRecord(self, arg): print(self.do_addRecord.__doc__) return - try: + try: if args.loadID: self.g2_module.addRecord(args.dataSourceCode, args.recordID, args.jsonData, args.loadID) else: @@ -1114,7 +1119,7 @@ def do_addRecord(self, arg): printWithNewLine('Record added.') except G2Exception.G2Exception as err: print(err) - + def do_addRecordWithInfo(self, arg): '\nAdd record with returned info: addRecordWithInfo [-l -f ]\n' @@ -1133,7 +1138,7 @@ def do_addRecordWithInfo(self, arg): if args.flags: flags = int(args.flags) - response = bytearray() + response = bytearray() self.g2_module.addRecordWithInfo(args.dataSourceCode, args.recordID, args.jsonData,response,loadId=loadID,flags=flags) if response: print('{}'.format(response.decode())) @@ -1152,8 +1157,8 @@ def do_addRecordWithReturnedRecordID(self, arg): print(self.do_addRecordWithReturnedRecordID.__doc__) return - try: - recordID = bytearray() + try: + recordID = bytearray() if args.loadID: self.g2_module.addRecordWithReturnedRecordID(args.dataSourceCode, recordID, args.jsonData, args.loadID) else: @@ -1164,7 +1169,7 @@ def do_addRecordWithReturnedRecordID(self, arg): print('\nNo record ID!\n') except G2Exception.G2Exception as err: print(err) - + def do_addRecordWithInfoWithReturnedRecordID(self, arg): '\nAdd record with returned record ID: addRecordWithInfoWithReturnedRecordID [-l ] [-f ]\n' @@ -1175,9 +1180,9 @@ def do_addRecordWithInfoWithReturnedRecordID(self, arg): print(self.do_addRecordWithInfoWithReturnedRecordID.__doc__) return - try: - recordID = bytearray() - info = bytearray() + try: + recordID = bytearray() + info = bytearray() if args.loadID: self.g2_module.addRecordWithInfoWithReturnedRecordID(args.dataSourceCode, args.jsonData, args.flags, recordID, info, args.loadID) else: @@ -1204,12 +1209,12 @@ def do_reevaluateRecord(self, arg): print(self.do_reevaluateRecord.__doc__) return - try: + try: self.g2_module.reevaluateRecord(args.dataSourceCode, args.recordID, args.flags) printWithNewLine('Record reevaluated.') except G2Exception.G2Exception as err: print(err) - + def do_reevaluateRecordWithInfo(self, arg): '\n Reevaluate record with returned info: reevaluateRecordWithInfo [-f flags]\n' @@ -1225,8 +1230,8 @@ def do_reevaluateRecordWithInfo(self, arg): if args.flags: flags = int(args.flags) - response = bytearray() - self.g2_module.reevaluateRecordWithInfo(args.dataSourceCode,args.recordID,response,flags=flags) + response = bytearray() + self.g2_module.reevaluateRecordWithInfo(args.dataSourceCode,args.recordID,response,flags=flags) if response: print('{}'.format(response.decode())) else: @@ -1246,7 +1251,7 @@ def do_reevaluateEntity(self, arg): print(self.do_reevaluateEntity.__doc__) return - try: + try: self.g2_module.reevaluateEntity(args.entityID, args.flags) printWithNewLine('Entity reevaluated.') except G2Exception.G2Exception as err: @@ -1268,7 +1273,7 @@ def do_reevaluateEntityWithInfo(self, arg): flags = int(args.flags) response = bytearray() - self.g2_module.reevaluateEntityWithInfo(args.entityID,response,flags=flags) + self.g2_module.reevaluateEntityWithInfo(args.entityID,response,flags=flags) if response: print('{}'.format(response.decode())) else: @@ -1288,7 +1293,7 @@ def do_replaceRecord(self, arg): print(self.do_replaceRecord.__doc__) return - try: + try: if args.loadID: self.g2_module.replaceRecord(args.dataSourceCode, args.recordID, args.jsonData, args.loadID) else: @@ -1317,7 +1322,7 @@ def do_replaceRecordWithInfo(self, arg): response = bytearray() self.g2_module.replaceRecordWithInfo(args.dataSourceCode,args.recordID,args.jsonData,response,loadId=loadID,flags=flags) - + if response: print('{}'.format(response.decode())) else: @@ -1329,7 +1334,7 @@ def do_replaceRecordWithInfo(self, arg): def do_deleteRecord(self, arg): - '\nDelete record: deleteRecord [-l loadID]\n' + '\nDelete record: deleteRecord [-l loadID]\n' try: args = self.parser.parse_args(['recordDelete'] + parse(arg)) @@ -1337,7 +1342,7 @@ def do_deleteRecord(self, arg): print(self.do_deleteRecord.__doc__) return - try: + try: if args.loadID: self.g2_module.deleteRecord(args.dataSourceCode, args.recordID, args.loadID) else: @@ -1348,7 +1353,7 @@ def do_deleteRecord(self, arg): def do_deleteRecordWithInfo(self, arg): - '\nDelete record with returned info: deleteRecord [-l loadID -f flags]\n' + '\nDelete record with returned info: deleteRecord [-l loadID -f flags]\n' try: args = self.parser.parse_args(['recordDeleteWithInfo'] + parse(arg)) @@ -1363,7 +1368,7 @@ def do_deleteRecordWithInfo(self, arg): loadID = args.loadID if args.flags: flags = int(args.flags) - + response = bytearray() self.g2_module.deleteRecordWithInfo(args.dataSourceCode,args.recordID,response,loadId=args.loadID,flags=flags) @@ -1386,8 +1391,8 @@ def do_searchByAttributes(self, arg): print(self.do_searchByAttributes.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.searchByAttributes(args.jsonData,response) if response: print('{}'.format(response.decode())) @@ -1406,8 +1411,8 @@ def do_searchByAttributesV2(self, arg): print(self.do_searchByAttributesV2.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.searchByAttributesV2(args.jsonData,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1426,8 +1431,8 @@ def do_getEntityByEntityID(self, arg): print(self.do_getEntityByEntityID.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.getEntityByEntityID(args.entityID, response) if response: print('{}'.format(response.decode())) @@ -1446,8 +1451,8 @@ def do_getEntityByEntityIDV2(self, arg): print(self.do_getEntityByEntityIDV2.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.getEntityByEntityIDV2(args.entityID,args.flags,response) if response: @@ -1467,8 +1472,8 @@ def do_findPathByEntityID(self, arg): print(self.do_findPathByEntityID.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.findPathByEntityID(args.startEntityID,args.endEntityID,args.maxDegree,response) if response: print('{}'.format(response.decode())) @@ -1487,8 +1492,8 @@ def do_findPathByEntityIDV2(self, arg): print(self.do_findPathByEntityIDV2.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.findPathByEntityIDV2(args.startEntityID,args.endEntityID,args.maxDegree,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1507,8 +1512,8 @@ def do_findNetworkByEntityID(self, arg): print(self.do_findNetworkByEntityID.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.findNetworkByEntityID(args.entityList,args.maxDegree,args.buildOutDegree,args.maxEntities,response) if response: print('{}'.format(response.decode())) @@ -1527,8 +1532,8 @@ def do_findNetworkByEntityIDV2(self, arg): print(self.do_findNetworkByEntityIDV2.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_module.findNetworkByEntityIDV2(args.entityList,args.maxDegree,args.buildOutDegree,args.maxEntities,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1540,15 +1545,15 @@ def do_findNetworkByEntityIDV2(self, arg): def do_findPathExcludingByEntityID(self, arg): '\nFind path between two entities, with exclusions: findPathExcludingByEntityID \n' - + try: args = self.parser.parse_args(['findPathExcludingByEntityID'] + parse(arg)) except SystemExit: print(self.do_findPathExcludingByEntityID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findPathExcludingByEntityID(args.startEntityID,args.endEntityID,args.maxDegree,args.excludedEntities,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1560,15 +1565,15 @@ def do_findPathExcludingByEntityID(self, arg): def do_findPathIncludingSourceByEntityID(self, arg): '\nFind path between two entities that includes a watched dsrc list, with exclusions: findPathIncludingSourceByEntityID \n' - + try: args = self.parser.parse_args(['findPathIncludingSourceByEntityID'] + parse(arg)) except SystemExit: print(self.do_findPathIncludingSourceByEntityID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findPathIncludingSourceByEntityID(args.startEntityID,args.endEntityID,args.maxDegree,args.excludedEntities,args.requiredDsrcs,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1586,9 +1591,9 @@ def do_getEntityByRecordID(self, arg): except SystemExit: print(self.do_getEntityByRecordID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.getEntityByRecordID(args.dataSourceCode, args.recordID,response) if response: print('{}'.format(response.decode())) @@ -1600,15 +1605,15 @@ def do_getEntityByRecordID(self, arg): def do_getEntityByRecordIDV2(self, arg): '\nGet entity by record ID: getEntityByRecordIDV2 \n' - + try: args = self.parser.parse_args(['getEntityByRecordIDV2'] + parse(arg)) except SystemExit: print(self.do_getEntityByRecordIDV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.getEntityByRecordIDV2(args.dataSourceCode, args.recordID, args.flags,response) if response: print('{}'.format(response.decode())) @@ -1620,15 +1625,15 @@ def do_getEntityByRecordIDV2(self, arg): def do_findPathByRecordID(self, arg): '\nFind path between two records: findPathByRecordID \n' - + try: args = self.parser.parse_args(['findPathByRecordID'] + parse(arg)) except SystemExit: print(self.do_findPathByRecordID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findPathByRecordID(args.startDataSourceCode,args.startRecordID,args.endDataSourceCode,args.endRecordID,args.maxDegree,response) if response: print('{}'.format(response.decode())) @@ -1640,15 +1645,15 @@ def do_findPathByRecordID(self, arg): def do_findPathByRecordIDV2(self, arg): '\nFind path between two records: findPathByRecordIDV2 \n' - + try: args = self.parser.parse_args(['findPathByRecordIDV2'] + parse(arg)) except SystemExit: print(self.do_findPathByRecordIDV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findPathByRecordIDV2(args.startDataSourceCode,args.startRecordID,args.endDataSourceCode,args.endRecordID,args.maxDegree,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1660,15 +1665,15 @@ def do_findPathByRecordIDV2(self, arg): def do_findNetworkByRecordID(self, arg): '\nFind network between records: findNetworkByRecordID \n' - + try: args = self.parser.parse_args(['findNetworkByRecordID'] + parse(arg)) except SystemExit: print(self.do_findNetworkByRecordID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findNetworkByRecordID(args.recordList,args.maxDegree,args.buildOutDegree,args.maxEntities,response) if response: print('{}'.format(response.decode())) @@ -1680,15 +1685,15 @@ def do_findNetworkByRecordID(self, arg): def do_findNetworkByRecordIDV2(self, arg): '\nFind network between records: findNetworkByRecordIDV2 \n' - + try: args = self.parser.parse_args(['findNetworkByRecordIDV2'] + parse(arg)) except SystemExit: print(self.do_findNetworkByRecordIDV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findNetworkByRecordIDV2(args.recordList,args.maxDegree,args.buildOutDegree,args.maxEntities,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1700,15 +1705,15 @@ def do_findNetworkByRecordIDV2(self, arg): def do_whyEntityByRecordID(self, arg): '\nDetermine why a record is inside its entity: whyEntityByRecordID \n' - + try: args = self.parser.parse_args(['whyEntityByRecordID'] + parse(arg)) except SystemExit: print(self.do_whyEntityByRecordID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyEntityByRecordID(args.dataSourceCode,args.recordID,response) if response: print('{}'.format(response.decode())) @@ -1720,15 +1725,15 @@ def do_whyEntityByRecordID(self, arg): def do_whyEntityByRecordIDV2(self, arg): '\nDetermine why a record is inside its entity: whyEntityByRecordIDV2 \n' - + try: args = self.parser.parse_args(['whyEntityByRecordIDV2'] + parse(arg)) except SystemExit: print(self.do_whyEntityByRecordIDV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyEntityByRecordIDV2(args.dataSourceCode,args.recordID,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1740,15 +1745,15 @@ def do_whyEntityByRecordIDV2(self, arg): def do_whyEntityByEntityID(self, arg): '\nDetermine why records are inside an entity: whyEntityByEntityID \n' - + try: args = self.parser.parse_args(['whyEntityByEntityID'] + parse(arg)) except SystemExit: print(self.do_whyEntityByEntityID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyEntityByEntityID(args.entityID,response) if response: print('{}'.format(response.decode())) @@ -1760,15 +1765,15 @@ def do_whyEntityByEntityID(self, arg): def do_whyEntityByEntityIDV2(self, arg): '\nDetermine why records are inside an entity: whyEntityByEntityIDV2 \n' - + try: args = self.parser.parse_args(['whyEntityByEntityIDV2'] + parse(arg)) except SystemExit: print(self.do_whyEntityByEntityIDV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyEntityByEntityIDV2(args.entityID,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1780,15 +1785,15 @@ def do_whyEntityByEntityIDV2(self, arg): def do_whyEntities(self, arg): '\nDetermine how entities relate to each other: whyEntities \n' - + try: args = self.parser.parse_args(['whyEntities'] + parse(arg)) except SystemExit: print(self.do_whyEntities.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyEntities(args.entityID1,args.entityID2,response) if response: print('{}'.format(response.decode())) @@ -1800,15 +1805,15 @@ def do_whyEntities(self, arg): def do_whyEntitiesV2(self, arg): '\nDetermine how entities relate to each other: whyEntitiesV2 \n' - + try: args = self.parser.parse_args(['whyEntitiesV2'] + parse(arg)) except SystemExit: print(self.do_whyEntitiesV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyEntitiesV2(args.entityID1,args.entityID2,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1820,15 +1825,15 @@ def do_whyEntitiesV2(self, arg): def do_whyRecords(self, arg): '\nDetermine how two records relate to each other: whyRecords \n' - + try: args = self.parser.parse_args(['whyRecords'] + parse(arg)) except SystemExit: print(self.do_whyRecords.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyRecords(args.dataSourceCode1,args.recordID1,args.dataSourceCode2,args.recordID2,response) if response: print('{}'.format(response.decode())) @@ -1840,15 +1845,15 @@ def do_whyRecords(self, arg): def do_whyRecordsV2(self, arg): '\nDetermine how two records relate to each other: whyRecordsV2 \n' - + try: args = self.parser.parse_args(['whyRecordsV2'] + parse(arg)) except SystemExit: print(self.do_whyRecordsV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.whyRecordsV2(args.dataSourceCode1,args.recordID1,args.dataSourceCode2,args.recordID2,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1866,9 +1871,9 @@ def do_findPathExcludingByRecordID(self, arg): except SystemExit: print(self.do_findPathExcludingByRecordID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findPathExcludingByRecordID(args.startDataSourceCode,args.startRecordID,args.endDataSourceCode,args.endRecordID,args.maxDegree,args.excludedEntities,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1880,15 +1885,15 @@ def do_findPathExcludingByRecordID(self, arg): def do_findPathIncludingSourceByRecordID(self, arg): '\nFind path between two records that includes a watched dsrc list, with exclusions: findPathIncludingSourceByRecordID \n' - + try: args = self.parser.parse_args(['findPathIncludingSourceByRecordID'] + parse(arg)) except SystemExit: print(self.do_findPathIncludingSourceByRecordID.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.findPathIncludingSourceByRecordID(args.startDataSourceCode,args.startRecordID,args.endDataSourceCode,args.endRecordID,args.maxDegree,args.excludedEntities,args.requiredDsrcs,args.flags,response) if response: print('{}'.format(response.decode())) @@ -1900,15 +1905,15 @@ def do_findPathIncludingSourceByRecordID(self, arg): def do_getRecord(self, arg): '\nGet record for record ID : getRecord \n' - + try: args = self.parser.parse_args(['getRecord'] + parse(arg)) except SystemExit: print(self.do_getRecord.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.getRecord(args.dataSourceCode, args.recordID,response) if response: print('{}'.format(response.decode())) @@ -1920,15 +1925,15 @@ def do_getRecord(self, arg): def do_getRecordV2(self, arg): '\nGet record for record ID : getRecordV2 \n' - + try: args = self.parser.parse_args(['getRecordV2'] + parse(arg)) except SystemExit: print(self.do_getRecordV2.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.getRecordV2(args.dataSourceCode, args.recordID, args.flags,response) if response: print('{}'.format(response.decode())) @@ -1941,7 +1946,7 @@ def do_getRecordV2(self, arg): def do_countRedoRecords(self,arg): '\nCounts the number of records in the redo queue: countRedoRecords\n' - try: + try: recordCount = self.g2_module.countRedoRecords() print('Record Count: %d' % recordCount) except G2Exception.G2Exception as err: @@ -1950,7 +1955,7 @@ def do_countRedoRecords(self,arg): def do_processRedoRecord(self, arg): '\nProcess a redo record: processRedoRecord \n' - + try: response = bytearray() self.g2_module.processRedoRecord(response) @@ -1964,13 +1969,13 @@ def do_processRedoRecord(self, arg): def do_processRedoRecordWithInfo(self, arg): '\nProcess a redo record with returned info: processRedoRecordWithInfo [-f ]\n' - + try: args = self.parser.parse_args(['processRedoRecordWithInfo'] + parse(arg)) except SystemExit: print(self.do_processRedoRecordWithInfo.__doc__) return - + try: flags = int(inspect.signature(self.g2_module.processRedoRecordWithInfo).parameters['flags'].default) if args.flags: @@ -1993,14 +1998,14 @@ def do_processRedoRecordWithInfo(self, arg): def do_getEntityDetails(self, arg): '\nGet the profile of a resolved entity: getEntityDetails -e [-d]\n' - + try: args = self.parser.parse_args(['getEntityDetails'] + parse(arg)) except SystemExit: print(self.do_getEntityDetails.__doc__) return - try: - response = bytearray() + try: + response = bytearray() self.g2_diagnostic_module.getEntityDetails(args.entityID,args.includeInternalFeatures,response) if response: print('{}'.format(response.decode())) @@ -2012,15 +2017,15 @@ def do_getEntityDetails(self, arg): def do_getRelationshipDetails(self, arg): '\nGet the profile of a relationship: getRelationshipDetails -r [-d]\n' - + try: args = self.parser.parse_args(['getRelationshipDetails'] + parse(arg)) except SystemExit: print(self.do_getRelationshipDetails.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.getRelationshipDetails(args.relationshipID,args.includeInternalFeatures,response) if response: print('{}'.format(response.decode())) @@ -2032,15 +2037,15 @@ def do_getRelationshipDetails(self, arg): def do_getEntityResume(self, arg): '\nGet the related records for a resolved entity: getEntityResume \n' - + try: args = self.parser.parse_args(['getEntityResume'] + parse(arg)) except SystemExit: print(self.do_getEntityResume.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.getEntityResume(args.entityID,response) if response: print('{}'.format(response.decode())) @@ -2052,16 +2057,16 @@ def do_getEntityResume(self, arg): def do_getEntityListBySize(self, arg): '\nGet list of resolved entities of specified size: getEntityListBySize -s [-o ]\n' - + try: args = self.parser.parse_args(['getEntityListBySize'] + parse(arg)) except SystemExit: print(self.do_getEntityListBySize.__doc__) return - - try: + + try: sizedEntityHandle = self.g2_diagnostic_module.getEntityListBySize(args.entitySize) - response = bytearray() + response = bytearray() rowData = self.g2_diagnostic_module.fetchNextEntityBySize(sizedEntityHandle,response) resultString = b"" while rowData: @@ -2085,9 +2090,9 @@ def do_getEntityListBySizeV2(self, arg): except SystemExit: print(self.do_getEntityListBySizeV2.__doc__) return - try: + try: sizedEntityHandle = self.g2_diagnostic_module.getEntityListBySizeV2(args.entitySize) - response = bytearray() + response = bytearray() rowData = self.g2_diagnostic_module.fetchNextEntityBySizeV2(sizedEntityHandle,response) resultString = b"" while rowData: @@ -2102,17 +2107,19 @@ def do_getEntityListBySizeV2(self, arg): print('{}'.format(resultString.decode())) except G2Exception.G2Exception as err: print(err) + + def do_checkDBPerf(self,arg): '\nRun a check on the DB performance: checkDBPerf\n' - + try: args = self.parser.parse_args(['checkDBPerf'] + parse(arg)) except SystemExit: print(self.do_checkDBPerf.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.checkDBPerf(args.secondsToRun,response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -2121,9 +2128,9 @@ def do_checkDBPerf(self,arg): def do_getDataSourceCounts(self,arg): '\nGet record counts by data source and entity type: getDataSourceCounts\n' - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.getDataSourceCounts(response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -2132,15 +2139,15 @@ def do_getDataSourceCounts(self,arg): def do_getMappingStatistics(self,arg): '\nGet data source mapping statistics: getMappingStatistics [-d]\n' - + try: args = self.parser.parse_args(['getMappingStatistics'] + parse(arg)) except SystemExit: print(self.do_getMappingStatistics.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.getMappingStatistics(args.includeInternalFeatures,response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -2149,15 +2156,15 @@ def do_getMappingStatistics(self,arg): def do_getGenericFeatures(self,arg): '\nGet a list of generic values for a feature type: getGenericFeatures [-t ] [-m ]\n' - + try: args = self.parser.parse_args(['getGenericFeatures'] + parse(arg)) except SystemExit: print(self.do_getGenericFeatures.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.getGenericFeatures(args.featureType,args.maximumEstimatedCount,response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -2166,15 +2173,15 @@ def do_getGenericFeatures(self,arg): def do_getEntitySizeBreakdown(self,arg): '\nGet the number of entities of each entity size: getEntitySizeBreakdown [-m ] [-d]\n' - + try: args = self.parser.parse_args(['getEntitySizeBreakdown'] + parse(arg)) except SystemExit: print(self.do_getEntitySizeBreakdown.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.getEntitySizeBreakdown(args.minimumEntitySize,args.includeInternalFeatures,response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -2183,9 +2190,9 @@ def do_getEntitySizeBreakdown(self,arg): def do_getResolutionStatistics(self,arg): '\nGet resolution statistics: getResolutionStatistics\n' - - try: - response = bytearray() + + try: + response = bytearray() self.g2_diagnostic_module.getResolutionStatistics(response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -2194,13 +2201,13 @@ def do_getResolutionStatistics(self,arg): def do_findEntitiesByFeatureIDs(self, arg): '\nGet the entities for a list of features: findEntitiesByFeatureIDs \n' - + try: args = self.parser.parse_args(['findEntitiesByFeatureIDs'] + parse(arg)) except SystemExit: print(self.do_findEntitiesByFeatureIDs.__doc__) return - + try: response = bytearray() self.g2_diagnostic_module.findEntitiesByFeatureIDs(args.features,response) @@ -2214,9 +2221,9 @@ def do_findEntitiesByFeatureIDs(self, arg): def do_stats(self,arg): '\nGet engine workload statistics for last process: stats\n' - - try: - response = bytearray() + + try: + response = bytearray() self.g2_module.stats(response) print('{}'.format(response.decode())) except G2Exception.G2Exception as err: @@ -2224,17 +2231,17 @@ def do_stats(self,arg): def do_exportConfig(self,arg): - '\nExport the config: exportConfig [-o ]\n' - + '\nExport the config: exportConfig [-o ]\n' + try: args = self.parser.parse_args(['outputOptional'] + parse(arg)) except SystemExit: print(self.do_exportConfig.__doc__) return - - try: - response = bytearray() - configID = bytearray() + + try: + response = bytearray() + configID = bytearray() self.g2_module.exportConfig(response,configID) responseMsg = json.loads(response) if args.outputFile: @@ -2247,10 +2254,10 @@ def do_exportConfig(self,arg): def do_getActiveConfigID(self,arg): - '\nGet the config identifier: getActiveConfigID\n' - - try: - response = bytearray() + '\nGet the config identifier: getActiveConfigID\n' + + try: + response = bytearray() self.g2_module.getActiveConfigID(response) printResponse(response.decode()) except G2Exception.G2Exception as err: @@ -2258,10 +2265,10 @@ def do_getActiveConfigID(self,arg): def do_getRepositoryLastModifiedTime(self,arg): - '\nGet the last modified time of the datastore: getRepositoryLastModifiedTime\n' - - try: - response = bytearray() + '\nGet the last modified time of the datastore: getRepositoryLastModifiedTime\n' + + try: + response = bytearray() self.g2_module.getRepositoryLastModifiedTime(response) printResponse(response.decode()) except G2Exception.G2Exception as err: @@ -2270,15 +2277,15 @@ def do_getRepositoryLastModifiedTime(self,arg): def do_exportTokenLibrary(self,arg): '\nExport the token library: exportTokenLibrary [-o ]\n' - + try: args = self.parser.parse_args(['outputOptional'] + parse(arg)) except SystemExit: print(self.do_exportTokenLibrary.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_hasher_module.exportTokenLibrary(response) responseMsg = json.loads(response) if args.outputFile: @@ -2307,7 +2314,7 @@ def do_purgeRepository(self, arg): return else: printWithNewLines(f'INFO: Purging without prompting, --FORCEPURGE was specified!', 'S') - + except SystemExit: print(self.do_purgeRepository.__doc__) return @@ -2318,26 +2325,26 @@ def do_purgeRepository(self, arg): else: reset_resolver=True resolver_txt = '(and resetting resolver)' - - printWithNewLines(f'Purging the repository {resolver_txt}', 'B') + + printWithNewLines(f'Purging the repository {resolver_txt}', 'B') try: - self.g2_module.purgeRepository(reset_resolver) + self.g2_module.purgeRepository(reset_resolver) except G2Exception.G2Exception as err: printWithNewLines(f'G2Exception: {err}', 'B') def do_hashRecord(self, arg): '\nHash an entity record: hashRecord \n' - + try: args = self.parser.parse_args(['jsonOnly'] + parse(arg)) except SystemExit: print(self.do_hashRecord.__doc__) return - - try: - response = bytearray() + + try: + response = bytearray() self.g2_hasher_module.process(args.jsonData,response) if response: print('{}'.format(response.decode())) @@ -2349,20 +2356,20 @@ def do_hashRecord(self, arg): def do_hashFile(self, arg): '\nHash a file of entity records: hashFile [-o ]\n' - + try: args = self.parser.parse_args(['inputFile'] + parse(arg)) except SystemExit: print(self.do_hashFile.__doc__) return - - try: + + try: print() if args.outputFile: with open(args.outputFile, 'w') as data_out: with open(args.inputFile.split("?")[0]) as data_in: for line in data_in: - response = bytearray() + response = bytearray() self.g2_hasher_module.process(line.strip(),response) hashedData = response.decode() data_out.write(hashedData) @@ -2370,7 +2377,7 @@ def do_hashFile(self, arg): else: with open(args.inputFile.split("?")[0]) as data_in : for line in data_in: - response = bytearray() + response = bytearray() self.g2_hasher_module.process(line.strip(),response) hashedData = response.decode() printWithNewLine(hashedData) @@ -2381,7 +2388,7 @@ def do_hashFile(self, arg): def do_license(self,arg): '\nGet the license information: license\n' - try: + try: response = self.g2_product_module.license() printWithNewLines(response, 'B') except G2Exception.G2Exception as err: @@ -2390,14 +2397,14 @@ def do_license(self,arg): def do_validateLicenseFile(self,arg): '\nValidate a license file: validateLicenseFile \n' - + try: args = self.parser.parse_args(['validateLicenseFile'] + parse(arg)) except SystemExit: print(self.do_validateLicenseFile.__doc__) return - - try: + + try: returnCode = self.g2_product_module.validateLicenseFile(args.licenseFilePath) if returnCode == 0: printWithNewLine('License validated') @@ -2410,7 +2417,7 @@ def do_validateLicenseFile(self,arg): def do_version(self,arg): '\nGet the version information: version\n' - try: + try: response = json.dumps(json.loads(self.g2_product_module.version())) print('\nG2 version:') printWithNewLine(response) @@ -2420,8 +2427,8 @@ def do_version(self,arg): def do_getPhysicalCores(self,arg): '\nGet the number of physical cores: getPhysicalCores\n' - - try: + + try: numCores = self.g2_diagnostic_module.getPhysicalCores() printWithNewLine('\nPhysical Cores: %d' % numCores) except G2Exception.G2Exception as err: @@ -2430,8 +2437,8 @@ def do_getPhysicalCores(self,arg): def do_getLogicalCores(self,arg): '\nGet the number of logical cores: getLogicalCores\n' - - try: + + try: numCores = self.g2_diagnostic_module.getLogicalCores() printWithNewLine('\nLogical Cores: %d' % numCores) except G2Exception.G2Exception as err: @@ -2440,8 +2447,8 @@ def do_getLogicalCores(self,arg): def do_getTotalSystemMemory(self,arg): '\nGet the total system memory: getTotalSystemMemory\n' - - try: + + try: memory = self.g2_diagnostic_module.getTotalSystemMemory() printWithNewLine('\nTotal System Memory: %d' % memory) except G2Exception.G2Exception as err: @@ -2451,7 +2458,7 @@ def do_getTotalSystemMemory(self,arg): def do_getAvailableMemory(self,arg): '\nGet the available memrory: getAvailableMemory\n' - try: + try: memory = self.g2_diagnostic_module.getAvailableMemory() printWithNewLine('\nAvailable Memory: %d' % memory) except G2Exception.G2Exception as err: @@ -2465,15 +2472,15 @@ def do_histDedupe(self, arg): if self.histAvail: if input('\nThis will de-duplicate both this session history and the history file, are you sure? (y/n) ') in ['y','Y', 'yes', 'YES']: - + with open(self.histFileName) as hf: linesIn = (line.rstrip() for line in hf) uniqLines = OrderedDict.fromkeys( line for line in linesIn if line ) - + readline.clear_history() for ul in uniqLines: readline.add_history(ul) - + printWithNewLines('Session history and history file both deduplicated.', 'B') else: print() @@ -2537,8 +2544,8 @@ def do_timer(self, arg): def parse(argumentString): 'Parses an argument list into a logical set of argument strings' - return shlex.split(argumentString) + return shlex.split(argumentString) def printWithNewLine(arg): print(arg + '\n') @@ -2547,21 +2554,21 @@ def printWithNewLine(arg): def printWithNewLines(ln, pos=''): pos = pos.upper() - if pos == 'S' or pos == 'START' : + + if pos == 'S' or pos == 'START': print(f'\n{ln}') - elif pos == 'E' or pos == 'END' : + elif pos == 'E' or pos == 'END': print(f'{ln}\n') - elif pos == 'B' or pos == 'BOTH' : + elif pos == 'B' or pos == 'BOTH': print(f'\n{ln}\n') else: - print(f'{ln}') + print(f'{ln}') def printResponse(response): print('\n' + response + '\n') - if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -2573,7 +2580,7 @@ def printResponse(response): first_loop = True restart = False - + # If ini file isn't specified try and locate it with G2Paths ini_file_name = pathlib.Path(G2Paths.get_G2Module_ini_path()) if not args.iniFile else pathlib.Path(args.iniFile[0]).resolve() G2Paths.check_file_exists_and_readable(ini_file_name) @@ -2592,16 +2599,16 @@ def printResponse(response): cmd_obj = G2CmdShell(args.debugTrace, args.histDisable, ini_file_name) cmd_obj.fileloop(args.fileToProcess) - # Start command shell + # Start command shell else: - # Don't use args.debugTrace here we may need to restart + # Don't use args.debugTrace here, may need to restart debug_trace = args.debugTrace while first_loop or restart: # Have we been in the command shell already and are trying to quit? Used for restarting if 'cmd_obj' in locals() and cmd_obj.ret_quit(): - break + break cmd_obj = G2CmdShell(debug_trace, args.histDisable, ini_file_name) cmd_obj.cmdloop() diff --git a/g2/python/G2ConfigTables.py b/g2/python/G2ConfigTables.py index 4a60322..14f14f3 100644 --- a/g2/python/G2ConfigTables.py +++ b/g2/python/G2ConfigTables.py @@ -4,12 +4,6 @@ import os import json -#--optional imports -try: import pyodbc -except: pass -try: import sqlite3 -except: pass - #====================== class G2ConfigTables: #====================== @@ -33,32 +27,35 @@ def loadConfig(self, tableName): for rowNode in tableNode: cfgNodeEntry = {} if tableName.upper() == 'CFG_DSRC': - cfgNodeEntry['ID'] = rowNode['DSRC_ID']; - cfgNodeEntry['DSRC_CODE'] = rowNode['DSRC_CODE']; - cfgDict[cfgNodeEntry['DSRC_CODE']] = cfgNodeEntry + cfgNodeEntry['ID'] = rowNode['DSRC_ID']; + cfgNodeEntry['DSRC_CODE'] = rowNode['DSRC_CODE']; + cfgDict[cfgNodeEntry['DSRC_CODE']] = cfgNodeEntry elif tableName.upper() == 'CFG_ETYPE': - cfgNodeEntry['ID'] = rowNode['ETYPE_ID']; - cfgNodeEntry['ETYPE_CODE'] = rowNode['ETYPE_CODE']; - cfgDict[cfgNodeEntry['ETYPE_CODE']] = cfgNodeEntry + cfgNodeEntry['ID'] = rowNode['ETYPE_ID']; + cfgNodeEntry['ETYPE_CODE'] = rowNode['ETYPE_CODE']; + cfgDict[cfgNodeEntry['ETYPE_CODE']] = cfgNodeEntry elif tableName.upper() == 'CFG_FTYPE': - cfgNodeEntry['ID'] = rowNode['FTYPE_ID']; - cfgNodeEntry['FTYPE_CODE'] = rowNode['FTYPE_CODE']; - cfgNodeEntry['DERIVED'] = rowNode['DERIVED']; - cfgDict[cfgNodeEntry['FTYPE_CODE']] = cfgNodeEntry + cfgNodeEntry['ID'] = rowNode['FTYPE_ID']; + cfgNodeEntry['FTYPE_CODE'] = rowNode['FTYPE_CODE']; + cfgNodeEntry['FTYPE_FREQ'] = rowNode['FTYPE_FREQ']; + cfgNodeEntry['FTYPE_EXCL'] = rowNode['FTYPE_EXCL']; + cfgNodeEntry['FTYPE_STAB'] = rowNode['FTYPE_STAB']; + cfgNodeEntry['DERIVED'] = rowNode['DERIVED']; + cfgDict[cfgNodeEntry['FTYPE_CODE']] = cfgNodeEntry elif tableName.upper() == 'CFG_ERRULE': - cfgNodeEntry['ID'] = rowNode['ERRULE_ID']; - cfgNodeEntry['ERRULE_CODE'] = rowNode['ERRULE_CODE']; - cfgNodeEntry['REF_SCORE'] = rowNode['REF_SCORE']; - cfgNodeEntry['RTYPE_ID'] = rowNode['RTYPE_ID']; - cfgDict[cfgNodeEntry['ID']] = cfgNodeEntry + cfgNodeEntry['ID'] = rowNode['ERRULE_ID']; + cfgNodeEntry['ERRULE_CODE'] = rowNode['ERRULE_CODE']; + cfgNodeEntry['REF_SCORE'] = rowNode['REF_SCORE']; + cfgNodeEntry['RTYPE_ID'] = rowNode['RTYPE_ID']; + cfgDict[cfgNodeEntry['ID']] = cfgNodeEntry elif tableName.upper() == 'CFG_ATTR': - cfgNodeEntry['ATTR_ID'] = rowNode['ATTR_ID']; - cfgNodeEntry['ATTR_CODE'] = rowNode['ATTR_CODE']; - cfgNodeEntry['ATTR_CLASS'] = rowNode['ATTR_CLASS']; - cfgNodeEntry['FTYPE_CODE'] = rowNode['FTYPE_CODE']; - cfgNodeEntry['FELEM_CODE'] = rowNode['FELEM_CODE']; - cfgNodeEntry['FELEM_REQ'] = rowNode['FELEM_REQ']; - cfgDict[cfgNodeEntry['ATTR_CODE']] = cfgNodeEntry + cfgNodeEntry['ATTR_ID'] = rowNode['ATTR_ID']; + cfgNodeEntry['ATTR_CODE'] = rowNode['ATTR_CODE']; + cfgNodeEntry['ATTR_CLASS'] = rowNode['ATTR_CLASS']; + cfgNodeEntry['FTYPE_CODE'] = rowNode['FTYPE_CODE']; + cfgNodeEntry['FELEM_CODE'] = rowNode['FELEM_CODE']; + cfgNodeEntry['FELEM_REQ'] = rowNode['FELEM_REQ']; + cfgDict[cfgNodeEntry['ATTR_CODE']] = cfgNodeEntry else: return None return cfgDict diff --git a/g2/python/G2ConfigTool.py b/g2/python/G2ConfigTool.py index cb7bc16..a581b70 100644 --- a/g2/python/G2ConfigTool.py +++ b/g2/python/G2ConfigTool.py @@ -5,12 +5,15 @@ import glob import json import os +import pathlib import platform import re +import shlex import sys import textwrap import traceback from collections import OrderedDict +from contextlib import suppress from datetime import datetime from shutil import copyfile @@ -18,12 +21,12 @@ try: from G2IniParams import G2IniParams + from G2Engine import G2Engine from G2Health import G2Health - from G2Database import G2Database from G2ConfigMgr import G2ConfigMgr from G2Config import G2Config import G2Exception -except: +except Exception: pass try: @@ -32,15 +35,20 @@ except ImportError: readline = None +try: + from pygments import highlight, lexers, formatters +except ImportError: + pass + + class G2CmdShell(cmd.Cmd, object): #Override function from cmd module to make command completion case insensitive def completenames(self, text, *ignored): - dotext = 'do_'+text - return [a[3:] for a in self.get_names() if a.lower().startswith(dotext.lower())] + dotext = 'do_' + text + return [a[3:] for a in self.get_names() if a.lower().startswith(dotext.lower())] - - def __init__(self, g2_cfg_file, hist_disable, force_mode, file_to_process, ini_file, g2Dbo): + def __init__(self, g2_cfg_file, hist_disable, force_mode, file_to_process, ini_file): cmd.Cmd.__init__(self) # Cmd Module settings @@ -48,10 +56,14 @@ def __init__(self, g2_cfg_file, hist_disable, force_mode, file_to_process, ini_f self.prompt = '(g2cfg) ' self.ruler = '-' self.doc_header = 'Configuration Commands' - self.misc_header = 'Help Topics (help )' + self.misc_header = 'Help Topics (help )' self.undoc_header = 'Misc Commands' self.__hidden_methods = ('do_shell', 'do_EOF', 'do_help') + self.g2_module = G2Engine() + self.g2_configmgr = G2ConfigMgr() + self.g2_config = G2Config() + # Set flag to know if running an interactive command shell or reading from file self.isInteractive = True @@ -63,25 +75,31 @@ def __init__(self, g2_cfg_file, hist_disable, force_mode, file_to_process, ini_f self.g2ConfigFileUsed = False self.configUpdated = False self.g2configFile = g2_cfg_file - self.iniFileName = G2Paths.get_G2Module_ini_path() if not ini_file else ini_file - self.getConfig() - - self.g2Dbo = g2Dbo + self.ini_file_name = ini_file # Processing input file self.forceMode = force_mode self.fileToProcess = file_to_process self.attributeClassList = ('NAME', 'ATTRIBUTE', 'IDENTIFIER', 'ADDRESS', 'PHONE', 'RELATIONSHIP', 'OTHER') - self.lockedFeatureList = ('NAME','ADDRESS', 'PHONE', 'DOB', 'REL_LINK', 'REL_ANCHOR', 'REL_POINTER') + self.lockedFeatureList = ('NAME', 'ADDRESS', 'PHONE', 'DOB', 'REL_LINK', 'REL_ANCHOR', 'REL_POINTER') self.doDebug = False + # Setup for pretty printing + self.prettyPrint = True + self.pygmentsInstalled = True if 'pygments' in sys.modules else False + # Readline and history self.readlineAvail = True if 'readline' in sys.modules else False self.histDisable = hist_disable self.histCheck() + self.parser = argparse.ArgumentParser(prog='', add_help=False) + self.subparsers = self.parser.add_subparsers() + + getConfig_parser = self.subparsers.add_parser('getConfig', usage=argparse.SUPPRESS) + getConfig_parser.add_argument('configID', type=int) def getConfig(self): ''' Get configutation from database or set default one if not found ''' @@ -96,84 +114,68 @@ def getConfig(self): self.cfgData = json.load(open(self.g2configFile), encoding="utf-8") except ValueError as e: print(f'\nERROR: {self.g2configFile} doesn\'t appear to be valid JSON!') - print(f'ERROR: {e}\n') + print(f' {e}\n') sys.exit(1) else: # Get the current configuration from the Senzing database - iniParams = iniParamCreator.getJsonINIParams(self.iniFileName) - g2ConfigMgr = G2ConfigMgr() - g2ConfigMgr.initV2('pyG2ConfigMgr', iniParams, False) - defaultConfigID = bytearray() - g2ConfigMgr.getDefaultConfigID(defaultConfigID) + self.g2_configmgr.getDefaultConfigID(defaultConfigID) # If a default config isn't found, create a new default configuration if not defaultConfigID: - print('\nWARN: No default config stored in the database, see: https://senzing.zendesk.com/hc/en-us/articles/360036587313') + print('\nWARNING: No default config stored in the database, see: https://senzing.zendesk.com/hc/en-us/articles/360036587313') print('\nINFO: Adding a new default configuration to the database...') - g2_config = G2Config() - - g2_config.initV2('pyG2Config', iniParams, False) - config_handle = g2_config.create() + config_handle = self.g2_config.create() config_default = bytearray() - g2_config.save(config_handle, config_default) + self.g2_config.save(config_handle, config_default) config_string = config_default.decode() # Persist new default config to Senzing Repository try: - addconfig_id = bytearray() - g2ConfigMgr.addConfig(config_string, 'New default configuration added by G2ConfigTool.', addconfig_id) - g2ConfigMgr.setDefaultConfigID(addconfig_id) - + self.g2ConfigMgr.addConfig(config_string, 'New default configuration added by G2ConfigTool.', addconfig_id) + self.g2ConfigMgr.setDefaultConfigID(addconfig_id) except G2Exception.G2ModuleGenericException: raise - g2_config.destroy() - + self.destroyEngines() + sys.exit(1) else: - config_current = bytearray() - g2ConfigMgr.getConfig(defaultConfigID, config_current) + self.g2_configmgr.getConfig(defaultConfigID, config_current) config_string = config_current.decode() self.cfgData = json.loads(config_string) - g2ConfigMgr.destroy() - + self.configUpdated = False def do_quit(self, arg): - if self.configUpdated and input('\nThere are unsaved changes, would you like to save first? (y/n) ') in ['y','Y', 'yes', 'YES']: - self.do_save(self) + if self.configUpdated and input('\nThere are unsaved changes, would you like to save first? (y/n) ') in ['y', 'Y', 'yes', 'YES']: + self.do_save(self) return True - def do_exit(self, arg): self.do_quit(self) return True - def do_EOF(self, line): return True - def emptyline(self): return def default(self, line): - printWithNewLines(f'ERROR: Unknown command, type help to list available commands.', 'B') + printWithNewLines('ERROR: Unknown command, type help to list available commands.', 'B') return - - # ----------------------------- def cmdloop(self): while True: @@ -182,11 +184,11 @@ def cmdloop(self): break except KeyboardInterrupt: if self.configUpdated: - if input('\n\nThere are unsaved changes, would you like to save first? (y/n) ') in ['y','Y', 'yes', 'YES']: + if input('\n\nThere are unsaved changes, would you like to save first? (y/n) ') in ['y', 'Y', 'yes', 'YES']: self.do_save(self) break - if input('\nAre you sure you want to exit? (y/n) ') in ['y','Y', 'yes', 'YES']: + if input('\nAre you sure you want to exit? (y/n) ') in ['y', 'Y', 'yes', 'YES']: break else: print() @@ -197,32 +199,66 @@ def cmdloop(self): for item in traceback.format_tb(traceback_): printWithNewLines(item) + def initEngines(self, init_msg=True): + + if init_msg: + printWithNewLines('Initializing Senzing engines...', 'B') + + try: + self.g2_module.initV2('pyG2E', g2module_params, False) + self.g2_configmgr.initV2('pyG2ConfigMgr', g2module_params, False) + self.g2_config.initV2('pyG2Config', g2module_params, False) + except G2Exception.G2Exception as ex: + printWithNewLines(f'ERROR: {ex}', 'B') + # Clean up before exiting + self.destroyEngines() + sys.exit(1) + + # Re-read config after a save + if self.configUpdated: + self.getConfig() + + def destroyEngines(self): + + with suppress(Exception): + self.g2_module.destroy() + self.g2_configmgr.destroy() + self.g2_config.destroy() + def preloop(self): - printWithNewLines('Welcome to G2Config Tool. Type help or ? to list commands.', 'B') + self.initEngines() + self.getConfig() + printWithNewLines('Welcome to G2Config Tool. Type help or ? to list commands.', 'E') def postloop(self): - pass + self.destroyEngines() + + def do_prettyPrint(self, arg): - #Hide functions from available list of Commands. Seperate help sections for some + if self.prettyPrint: + self.prettyPrint = False + printWithNewLines('JSON pretty print is now off', 'B') + else: + self.prettyPrint = True + printWithNewLines('JSON pretty print is now on', 'B') + + # Hide functions from available list of Commands. Seperate help sections for some def get_names(self): return [n for n in dir(self.__class__) if n not in self.__hidden_methods] - def help_KnowledgeCenter(self): printWithNewLines(textwrap.dedent('''\ - Senzing Knowledge Center: https://senzing.zendesk.com/hc/en-us '''), 'S') - def help_Support(self): printWithNewLines(textwrap.dedent('''\ - Senzing Support Request: https://senzing.zendesk.com/hc/en-us/requests/new '''), 'S') - def help_Arguments(self): printWithNewLines(textwrap.dedent('''\ - Argument values to specify are surrounded with < >, replace with your value @@ -234,13 +270,11 @@ def help_Arguments(self): addAttribute {"attribute": "myNewAttribute"} '''), 'S') - def help_Shell(self): printWithNewLines(textwrap.dedent('''\ - Run basic OS shell commands: ! '''), 'S') - def help_History(self): printWithNewLines(textwrap.dedent(f'''\ - Use shell like history, requires Python readline module. @@ -264,6 +298,19 @@ def help_History(self): - History file error: {self.histFileError} '''), 'S') + def help_PrettyPrint(self): + printWithNewLines(textwrap.dedent(f'''\ + - Toggles pretty printing on/off for JSON output + + - Pretty print enabled: {self.prettyPrint} + '''), 'S') + + def help_PrettyPrintColor(self): + printWithNewLines(textwrap.dedent(f'''\ + - To colorize pretty printed JSON output install python Pygments if not installed: https://pypi.org/project/Pygments/ + + - Python Pygments installed: {self.pygmentsInstalled} + '''), 'S') def do_shell(self, line): '\nRun OS shell commands: !\n' @@ -271,7 +318,6 @@ def do_shell(self, line): output = os.popen(line).read() print(output) - def histCheck(self): ''' ''' @@ -311,15 +357,14 @@ def histCheck(self): self.histFileError = None self.histAvail = True - def do_histDedupe(self, arg): if self.histAvail: - if input('\nThis will de-duplicate both this session history and the history file, are you sure? (y/n) ') in ['y','Y', 'yes', 'YES']: + if input('\nThis will de-duplicate both this session history and the history file, are you sure? (y/n) ') in ['y', 'Y', 'yes', 'YES']: with open(self.histFileName) as hf: linesIn = (line.rstrip() for line in hf) - uniqLines = OrderedDict.fromkeys( line for line in linesIn if line ) + uniqLines = OrderedDict.fromkeys(line for line in linesIn if line) readline.clear_history() for ul in uniqLines: @@ -331,11 +376,10 @@ def do_histDedupe(self, arg): else: printWithNewLines('History isn\'t available in this session.', 'B') - def do_histClear(self, arg): if self.histAvail: - if input('\nThis will clear both this session history and the history file, are you sure? (y/n) ') in ['y','Y', 'yes', 'YES']: + if input('\nThis will clear both this session history and the history file, are you sure? (y/n) ') in ['y', 'Y', 'yes', 'YES']: readline.clear_history() readline.write_history_file(self.histFileName) printWithNewLines('Session history and history file both cleared.', 'B') @@ -344,7 +388,6 @@ def do_histClear(self, arg): else: printWithNewLines('History isn\'t available in this session.', 'B') - def do_histShow(self, arg): if self.histAvail: @@ -355,9 +398,12 @@ def do_histShow(self, arg): else: printWithNewLines('History isn\'t available in this session.', 'B') - def fileloop(self): + # Get initial config + self.initEngines(init_msg=False) + self.getConfig() + # Set flag to know running an interactive command shell or not self.isInteractive = False @@ -366,14 +412,14 @@ def fileloop(self): with open(self.fileToProcess) as data_in: for line in data_in: line = line.strip() - if len(line) > 0 and line[0:1] not in ('#','-','/'): + if len(line) > 0 and line[0:1] not in ('#', '-', '/'): #*args allows for empty list if there are no args (read_cmd, *args) = line.split() process_cmd = f'do_{read_cmd}' printWithNewLines(f'----- {read_cmd} -----', 'S') printWithNewLines(f'{line}', 'S') - if process_cmd == 'do_save' and not save_detected : + if process_cmd == 'do_save' and not save_detected: save_detected = True if process_cmd not in dir(self): @@ -389,13 +435,12 @@ def fileloop(self): if not save_detected and self.configUpdated: if not self.forceMode: - if input('\nWARN: No save command was issued would you like to save now? ') in ['y','Y', 'yes', 'YES']: + if input('\nWARNING: No save command was issued would you like to save now? ') in ['y', 'Y', 'yes', 'YES']: self.do_save(self) print() return - printWithNewLines('WARN: Configuration changes were made but have not been saved!', 'B') - + printWithNewLines('WARNING: Configuration changes were made but have not been saved!', 'B') def getRecord(self, table, field, value): @@ -413,8 +458,7 @@ def getRecord(self, table, field, value): return self.cfgData['G2_CONFIG'][table][i] return None - - def getRecordList(self, table, field = None, value = None): + def getRecordList(self, table, field=None, value=None): recordList = [] for i in range(len(self.cfgData['G2_CONFIG'][table])): @@ -425,6 +469,61 @@ def getRecordList(self, table, field = None, value = None): recordList.append(self.cfgData['G2_CONFIG'][table][i]) return recordList +# ===== General config ===== + + def do_getActiveConfigID(self, arg): + '\nGet the config identifier: getActiveConfigID\n' + + response = bytearray() + + try: + self.g2_module.getActiveConfigID(response) + printResponse(response) + except G2Exception.G2Exception as err: + printWithNewLines(err, 'B') + + def do_getConfig(self, arg): + '\nGet the config: getConfig \n' + + try: + args = self.parser.parse_args(['getConfig'] + parse(arg)) + except SystemExit: + print(self.do_getConfig.__doc__) + return + + response = bytearray() + + try: + self.g2_configmgr.getConfig(args.configID, response) + self.printJsonResponse(response) + except G2Exception.G2Exception as err: + print(err) + + def do_getConfigList(self, arg): + '\nGet a list of known configs: getConfigList \n' + + response = bytearray() + + try: + self.g2_configmgr.getConfigList(response) + self.printJsonResponse(response) + except G2Exception.G2Exception as err: + print(err) + + def do_getConfigSection(self, arg): + '\n\tGet a single section from the current config: getConfigSection
\n' + + if not argCheck('do_getConfigSection', arg, self.do_getConfigSection.__doc__): + return + + try: + section = json.dumps(self.cfgData["G2_CONFIG"][arg]) + except KeyError: + printWithNewLines(f'{arg} is an unknown section in the configuration!', 'B') + return + + self.printJsonResponse(section) + # ===== global commands ===== @@ -433,7 +532,7 @@ def do_configReload(self, arg): # Check if config has unsaved changes if self.configUpdated: - if input('\nYou have unsaved changes, are you sure you want to discard them? (y/n) ') not in ['y','Y', 'yes', 'YES']: + if input('\nYou have unsaved changes, are you sure you want to discard them? (y/n) ') not in ['y', 'Y', 'yes', 'YES']: printWithNewLines('\nConfiguration wasn\'t reloaded. Your changes remain but are still unsaved.\n') return @@ -443,7 +542,6 @@ def do_configReload(self, arg): printWithNewLines('Config has been reloaded.', 'B') - def do_save(self, args): '\n\tSave changes to the config\n' @@ -451,10 +549,10 @@ def do_save(self, args): # If not accepting file commands without prompts and not using older style config file if not self.forceMode and not self.g2ConfigFileUsed: - printWithNewLines('WARN: This will immediately update the current configuration in the Senzing repository with the current configuration!','B') - if input('Are you certain you wish to proceed and save changes? ') not in ['y','Y', 'yes', 'YES']: - printWithNewLines('Current configuration changes have not been saved!', 'B') - return + printWithNewLines('WARNING: This will immediately update the current configuration in the Senzing repository with the current configuration!', 'B') + if input('Are you certain you wish to proceed and save changes? (y/n) ') not in ['y', 'Y', 'yes', 'YES']: + printWithNewLines('Current configuration changes have not been saved!', 'B') + return if self.g2ConfigFileUsed: @@ -469,48 +567,46 @@ def do_save(self, args): return else: with open(self.g2configFile, 'w') as fp: - json.dump(self.cfgData, fp, indent = 4, sort_keys = True) + json.dump(self.cfgData, fp, indent=4, sort_keys=True) printWithNewLines(f'Configuration saved to {self.g2configFile}.', 'B') self.configUpdated = False else: try: - iniParamCreator = G2IniParams() - iniParams = iniParamCreator.getJsonINIParams(iniFileName) - g2ConfigMgr = G2ConfigMgr() - g2ConfigMgr.initV2('pyG2ConfigMgr', iniParams, False) - newConfigId = bytearray() - g2ConfigMgr.addConfig(json.dumps(self.cfgData), 'Updated by G2ConfigTool', newConfigId) - g2ConfigMgr.setDefaultConfigID(newConfigId) - g2ConfigMgr.destroy() - except: - printWithNewLines('ERROR: Failed to save configuration to Senzing repository!', 'B') + self.g2_configmgr.addConfig(json.dumps(self.cfgData), 'Updated by G2ConfigTool', newConfigId) + self.g2_configmgr.setDefaultConfigID(newConfigId) + + except G2Exception.G2ModuleGenericException as err: + printWithNewLines('ERROR: Failed to save configuration to Senzing repository!', 'S') + printWithNewLines(f' {err}', 'E') else: printWithNewLines('Configuration saved to Senzing repository.', 'B') - self.configUpdated = False + # Reinit engines to pick up changes. This is good practice and will be needed when rewritten to fully use cfg APIs + # Don't disply init msg if not interactive (fileloop) + self.destroyEngines() + self.initEngines(init_msg=(True if self.isInteractive else False)) + self.configUpdated = False -# ===== Autocompleters for import/export ===== +# ===== Autocompleters ===== def complete_exportToFile(self, text, line, begidx, endidx): if re.match("exportToFile +", line): return self.pathCompletes(text, line, begidx, endidx, 'exportToFile') - def complete_importFromFile(self, text, line, begidx, endidx): if re.match("importFromFile +", line): return self.pathCompletes(text, line, begidx, endidx, 'importFromFile') - def pathCompletes(self, text, line, begidx, endidx, callingcmd): ''' Auto complete paths for commands that have a complete_ function ''' completes = [] - pathComp = line[len(callingcmd)+1:endidx] - fixed = line[len(callingcmd)+1:begidx] + pathComp = line[len(callingcmd) + 1:endidx] + fixed = line[len(callingcmd) + 1:begidx] for path in glob.glob(f'{pathComp}*'): path = path + os.sep if path and os.path.isdir(path) and path[-1] != os.sep else path @@ -518,6 +614,71 @@ def pathCompletes(self, text, line, begidx, endidx, callingcmd): return completes + # Auto completers for approproate commands + def complete_getAttribute(self, text, line, begidx, endidx): + return self.codes_completes('CFG_ATTR', 'ATTR_CODE', text) + + def complete_getFeature(self, text, line, begidx, endidx): + return self.codes_completes('CFG_FTYPE', 'FTYPE_CODE', text) + + def complete_getElement(self, text, line, begidx, endidx): + return self.codes_completes('CFG_FELEM', 'FELEM_CODE', text) + + def complete_getFragment(self, text, line, begidx, endidx): + return self.codes_completes('CFG_ERFRAG', 'ERFRAG_CODE', text) + + def complete_getRule(self, text, line, begidx, endidx): + return self.codes_completes('CFG_ERRULE', 'ERRULE_CODE', text) + + def complete_deleteAttribute(self, text, line, begidx, endidx): + return self.codes_completes('CFG_ATTR', 'ATTR_CODE', text) + + def complete_deleteDataSource(self, text, line, begidx, endidx): + return self.codes_completes('CFG_DSRC', 'DSRC_CODE', text) + + def complete_deleteElement(self, text, line, begidx, endidx): + return self.codes_completes('CFG_FELEM', 'FELEM_CODE', text) + + def complete_deleteEntityType(self, text, line, begidx, endidx): + return self.codes_completes('CFG_ETYPE', 'ETYPE_CODE', text) + + def complete_deleteFeature(self, text, line, begidx, endidx): + return self.codes_completes('CFG_FTYPE', 'FTYPE_CODE', text) + + def complete_deleteFeatureComparison(self, text, line, begidx, endidx): + return self.codes_completes('CFG_FTYPE', 'FTYPE_CODE', text) + + def complete_deleteFeatureDistinctCall(self, text, line, begidx, endidx): + return self.codes_completes('CFG_FTYPE', 'FTYPE_CODE', text) + + def complete_deleteFragment(self, text, line, begidx, endidx): + return self.codes_completes('CFG_ERFRAG', 'ERFRAG_CODE', text) + + def complete_deleteRule(self, text, line, begidx, endidx): + return self.codes_completes('CFG_ERRULE', 'ERRULE_CODE', text) + + def codes_completes(self, table, field, arg): + ''' Return codes for auto complete on get* delete* ''' + + # Build list each time to have latest even after an add*, delete* + codes_list = self.getRecordCodes(table, field) + + return [code for code in codes_list if code.lower().startswith(arg.lower())] + + def getRecordCodes(self, table, field): + ''' Return list of codes from config ''' + + code_list = [] + for i in range(len(self.cfgData['G2_CONFIG'][table])): + code_list.append(self.cfgData["G2_CONFIG"][table][i][field]) + + return code_list + + def complete_getConfigSection(self, text, line, begidx, endidx): + ''' ''' + return [section for section in self.cfgData["G2_CONFIG"].keys() if section.lower().startswith(text.lower())] + +# ===== Export / Import ===== def do_exportToFile(self, arg): '\n\tExport the config to a file: exportToFile \n' @@ -527,7 +688,7 @@ def do_exportToFile(self, arg): try: with open(arg, 'w') as fp: - json.dump(self.cfgData, fp, indent = 4, sort_keys = True) + json.dump(self.cfgData, fp, indent=4, sort_keys=True) except OSError as ex: printWithNewLines(textwrap.dedent(f'''\ ERROR: Couldn\'t export to {arg}. @@ -537,7 +698,6 @@ def do_exportToFile(self, arg): else: printWithNewLines('Successfully exported!', 'B') - def do_importFromFile(self, arg): '\n\tImport a config from a file: importFromFile \n' @@ -545,7 +705,7 @@ def do_importFromFile(self, arg): return if self.configUpdated: - if input('\nYou have unsaved changes, are you sure you want to discard them? (y/n) ') not in ['y','Y', 'yes', 'YES']: + if input('\nYou have unsaved changes, are you sure you want to discard them? (y/n) ') not in ['y', 'Y', 'yes', 'YES']: printWithNewLines('Configuration wasn\'t imported. Your changes remain but are still unsaved.') return @@ -575,8 +735,9 @@ def do_verifyCompatibilityVersion(self, arg): else: if self.cfgData['G2_CONFIG']['CONFIG_BASE_VERSION']['COMPATIBILITY_VERSION']['CONFIG_VERSION'] != parmData['EXPECTEDVERSION']: - printWithNewLines('Compatibility version does not match specified value [%s]! Actual version is [%s].' % (parmData['EXPECTEDVERSION'],self.cfgData['G2_CONFIG']['CONFIG_BASE_VERSION']['COMPATIBILITY_VERSION']['CONFIG_VERSION']), 'B') - if self.isInteractive == False: + printWithNewLines('Compatibility version does not match specified value [%s]! Actual version is [%s].' % (parmData['EXPECTEDVERSION'], self.cfgData['G2_CONFIG']['CONFIG_BASE_VERSION']['COMPATIBILITY_VERSION']['CONFIG_VERSION']), 'B') + + if self.isInteractive is False: # throw an exception, so that we abort running scripts raise Exception('Incorrect compatibility version.') else: @@ -584,7 +745,6 @@ def do_verifyCompatibilityVersion(self, arg): return printWithNewLines('Compatibility version successfully verified!', 'B') - def do_updateCompatibilityVersion(self, arg): '\n\tupdateCompatibilityVersion {"fromVersion": "1", "toVersion": "2"}\n' @@ -605,7 +765,6 @@ def do_updateCompatibilityVersion(self, arg): self.configUpdated = True printWithNewLines('Compatibility version successfully changed!', 'B') - def do_getCompatibilityVersion(self, arg): '\n\tgetCompatibilityVersion\n' @@ -613,7 +772,7 @@ def do_getCompatibilityVersion(self, arg): compat_version = self.cfgData['G2_CONFIG']['CONFIG_BASE_VERSION']['COMPATIBILITY_VERSION']['CONFIG_VERSION'] printWithNewLines(f'Compatibility version: {compat_version}', 'B') except KeyError: - printWithNewLines(f'ERROR: Couldn\'t retrieve compatibility version', 'B') + printWithNewLines('ERROR: Couldn\'t retrieve compatibility version', 'B') # ===== Config section commands ===== @@ -639,7 +798,6 @@ def do_addConfigSection(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def do_addConfigSectionField(self, arg): '\n\taddConfigSectionField {"section": "","field": "","value": ""}\n' @@ -649,15 +807,15 @@ def do_addConfigSectionField(self, arg): try: parmData = dictKeysUpper(json.loads(arg)) - if not 'SECTION' in parmData or len(parmData['SECTION']) == 0: + if 'SECTION' not in parmData or len(parmData['SECTION']) == 0: raise ValueError('Config section name is required!') parmData['SECTION'] = parmData['SECTION'].upper() - if not 'FIELD' in parmData or len(parmData['FIELD']) == 0: + if 'FIELD' not in parmData or len(parmData['FIELD']) == 0: raise ValueError('Field name is required!') parmData['FIELD'] = parmData['FIELD'].upper() - if not 'VALUE' in parmData: + if 'VALUE' not in parmData: raise ValueError('Field value is required!') parmData['VALUE'] = parmData['VALUE'] @@ -684,14 +842,15 @@ def do_addConfigSectionField(self, arg): # ===== data Source commands ===== def do_listDataSources(self, arg): - '\n\tlistDataSources\n' + '\n\tlistDataSources [search_value]\n' print() - for dsrcRecord in sorted(self.getRecordList('CFG_DSRC'), key = lambda k: k['DSRC_ID']): + for dsrcRecord in sorted(self.getRecordList('CFG_DSRC'), key=lambda k: k['DSRC_ID']): + if arg and arg.lower() not in str(dsrcRecord).lower(): + continue print('{"id": %i, "dataSource": "%s"}' % (dsrcRecord['DSRC_ID'], dsrcRecord['DSRC_CODE'])) print() - def do_addDataSource(self, arg): '\n\taddDataSource {"dataSource": ""}\n' @@ -731,7 +890,6 @@ def do_addDataSource(self, arg): if self.doDebug: debug(newRecord) - def do_deleteDataSource(self, arg): '\n\tdeleteDataSource {"dataSource": ""}\n' @@ -746,7 +904,7 @@ def do_deleteDataSource(self, arg): else: if parmData['DATASOURCE'] == 'SEARCH': - printWithNewLine('Can\'t delete the SEARCH data source!') + printWithNewLines('Can\'t delete the SEARCH data source!', 'B') return deleteCnt = 0 @@ -755,6 +913,7 @@ def do_deleteDataSource(self, arg): del self.cfgData['G2_CONFIG']['CFG_DSRC'][i] deleteCnt += 1 self.configUpdated = True + if deleteCnt == 0: printWithNewLines('Record not found!', 'B') else: @@ -764,16 +923,18 @@ def do_deleteDataSource(self, arg): # ===== entity class commands ===== def do_listEntityClasses(self, arg): - '\n\tlistEntityClasses\n' + '\n\tlistEntityClasses [search_value]\n' print() - for eclassRecord in sorted(self.getRecordList('CFG_ECLASS'), key = lambda k: k['ECLASS_ID']): + for eclassRecord in sorted(self.getRecordList('CFG_ECLASS'), key=lambda k: k['ECLASS_ID']): + if arg and arg.lower() not in str(eclassRecord).lower(): + continue print('{"id": %i, "entityClass": "%s"}' % (eclassRecord['ECLASS_ID'], eclassRecord['ECLASS_CODE'])) print() ## ---------------------------- #def do_addEntityClass(self ,arg): - '\n\taddEntityClass {"entityClass": ""}\n' + # '\n\taddEntityClass {"entityClass": ""}\n' # # if not argCheck('addEntityClass', arg, self.do_addEntityClass.__doc__): # return @@ -844,15 +1005,16 @@ def do_listEntityClasses(self, arg): # ===== entity type commands ===== def do_listEntityTypes(self, arg): - '\n\tlistEntityTypes\n' + '\n\tlistEntityTypes [search_value]\n' print() - for etypeRecord in sorted(self.getRecordList('CFG_ETYPE'), key = lambda k: k['ETYPE_ID']): + for etypeRecord in sorted(self.getRecordList('CFG_ETYPE'), key=lambda k: k['ETYPE_ID']): + if arg and arg.lower() not in str(etypeRecord).lower(): + continue eclassRecord = self.getRecord('CFG_ECLASS', 'ECLASS_ID', etypeRecord['ECLASS_ID']) print('{"id": %i, "entityType":"%s", "class": "%s"}' % (etypeRecord['ETYPE_ID'], etypeRecord['ETYPE_CODE'], ('unknown' if not eclassRecord else eclassRecord['ECLASS_CODE']))) print() - def do_addEntityType(self, arg): '\n\taddEntityType {"entityType": ""}\n' @@ -901,7 +1063,6 @@ def do_addEntityType(self, arg): if self.doDebug: debug(newRecord) - def do_deleteEntityType(self, arg): '\n\tdeleteEntityType {"entityType": ""}\n' @@ -929,46 +1090,51 @@ def do_deleteEntityType(self, arg): # ===== feature commands ===== - def do_listFunctions(self, arg): - '\n\tlistFunctions\n' + '\n\tlistFunctions [search_value]\n' print() for funcRecord in sorted(self.getRecordList('CFG_SFUNC'), key = lambda k: k['SFUNC_ID']): - print('{"type": "Standardization", "function": "%s"}' % (funcRecord['SFUNC_CODE'])) + if arg and arg.lower() not in str(funcRecord).lower(): + continue + print(f'{{"id": "{funcRecord["SFUNC_ID"]}", type": "Standardization", "function": "{funcRecord["SFUNC_CODE"]}"}}') + print() for funcRecord in sorted(self.getRecordList('CFG_EFUNC'), key = lambda k: k['EFUNC_ID']): - print('{"type": "Expression", "function": "%s"}' % (funcRecord['EFUNC_CODE'])) + if arg and arg.lower() not in str(funcRecord).lower(): + continue + print(f'{{"id": "{funcRecord["EFUNC_ID"]}", type": "Expression", "function": "{funcRecord["EFUNC_CODE"]}"}}') + print() for funcRecord in sorted(self.getRecordList('CFG_CFUNC'), key = lambda k: k['CFUNC_ID']): - print('{"type": "Comparison", "function": "%s"}' % (funcRecord['CFUNC_CODE'])) - print() + if arg and arg.lower() not in str(funcRecord).lower(): + continue + print(f'{{"id": "{funcRecord["CFUNC_ID"]}", type": "Comparison", "function": "{funcRecord["CFUNC_CODE"]}"}}') + print() def do_listFeatureClasses(self, arg): - '\n\tlistFeatureClasses\n' + '\n\tlistFeatureClasses [search_value]\n' print() - for fclassRecord in sorted(self.getRecordList('CFG_FCLASS'), key = lambda k: k['FCLASS_ID']): + for fclassRecord in sorted(self.getRecordList('CFG_FCLASS'), key=lambda k: k['FCLASS_ID']): + if arg and arg.lower() not in str(fclassRecord).lower(): + continue print('{"id": %i, "class":"%s"}' % (fclassRecord['FCLASS_ID'], fclassRecord['FCLASS_CODE'])) print() - def do_listFeatures(self, arg): - '\n\tlistFeatures\n' + '\n\tlistFeatures [search_value]\n' #'\n\tlistFeatures\t\t(displays all features)\n' #\ #'listFeatures -n\t\t(to display new features only)\n' print() - for ftypeRecord in sorted(self.getRecordList('CFG_FTYPE'), key = lambda k: k['FTYPE_ID']): - if arg != '-n' or ftypeRecord['FTYPE_ID'] >= 1000: - featureJson = self.getFeatureJson(ftypeRecord) - print(featureJson) - if 'ERROR:' in featureJson: - print('Corrupted config! Delete this feature and re-add.') + for ftypeRecord in sorted(self.getRecordList('CFG_FTYPE'), key=lambda k: k['FTYPE_ID']): + if arg and arg.lower() not in str(ftypeRecord).lower(): + continue + printWithNewLines(self.getFeatureJson(ftypeRecord), 'E') print() - def do_getFeature(self, arg): '\n\tgetFeature {"feature": ""}\n' @@ -986,8 +1152,7 @@ def do_getFeature(self, arg): if not ftypeRecord: printWithNewLines('Feature %s not found!' % parmData['FEATURE'], 'B') else: - printWithNewLines(self.getFeatureJson(ftypeRecord), 'B') - + self.printJsonResponse(self.getFeatureJson(ftypeRecord)) def do_addFeatureComparisonElement(self, arg): '\n\taddFeatureComparisonElement {"feature": "", "element": ""}\n' @@ -1050,7 +1215,6 @@ def do_addFeatureComparisonElement(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def do_addFeatureDistinctCallElement(self, arg): '\n\taddFeatureDistinctCallElement {"feature": "", "element": ""}\n' @@ -1112,7 +1276,6 @@ def do_addFeatureDistinctCallElement(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def do_addFeatureComparison(self, arg): '\n\taddFeatureComparison {"feature": "", "comparison": "", "elementList": [""]\n\n\tNote the [ and ]') @@ -1138,7 +1301,7 @@ def do_addFeatureComparison(self, arg): return ftypeID = ftypeRecord['FTYPE_ID'] - cfuncID = 0 #--comparison function + cfuncID = 0 # --comparison function if 'COMPARISON' not in parmData or len(parmData['COMPARISON']) == 0: printWithNewLines('Comparison function not specified!', 'B') return @@ -1211,7 +1374,6 @@ def do_addFeatureComparison(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def do_deleteFeatureComparisonElement(self, arg): '\n\tdeleteFeatureComparisonElement {"feature": "", "element": ""}\n' @@ -1239,7 +1401,7 @@ def do_deleteFeatureComparisonElement(self, arg): return deleteCnt = 0 - for i in range(len(self.cfgData['G2_CONFIG']['CFG_FTYPE'])-1, -1, -1): + for i in range(len(self.cfgData['G2_CONFIG']['CFG_FTYPE']) -1, -1, -1): if self.cfgData['G2_CONFIG']['CFG_FTYPE'][i]['FTYPE_CODE'] == parmData['FEATURE']: for i1 in range(len(self.cfgData['G2_CONFIG']['CFG_CFCALL'])-1, -1, -1): if self.cfgData['G2_CONFIG']['CFG_CFCALL'][i1]['FTYPE_ID'] == ftypeRecord['FTYPE_ID']: @@ -1254,7 +1416,6 @@ def do_deleteFeatureComparisonElement(self, arg): else: printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_deleteFeatureComparison(self, arg): '\n\tdeleteFeatureComparison {"feature": ""}\n' @@ -1262,7 +1423,7 @@ def do_deleteFeatureComparison(self, arg): return try: - parmData = dictKeysUpper(json.loads(arg)) + parmData = dictKeysUpper(json.loads(arg)) if arg.startswith('{') else {"FEATURE": arg} parmData['FEATURE'] = parmData['FEATURE'].upper() except (ValueError, KeyError) as e: argError(arg, e) @@ -1294,7 +1455,6 @@ def do_deleteFeatureComparison(self, arg): else: printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_deleteFeatureDistinctCall(self, arg): '\n\tdeleteFeatureDistinctCall {"feature": ""}\n' @@ -1302,7 +1462,7 @@ def do_deleteFeatureDistinctCall(self, arg): return try: - parmData = dictKeysUpper(json.loads(arg)) + parmData = dictKeysUpper(json.loads(arg)) if arg.startswith('{') else {"FEATURE": arg} parmData['FEATURE'] = parmData['FEATURE'].upper() except (ValueError, KeyError) as e: argError(arg, e) @@ -1334,7 +1494,6 @@ def do_deleteFeatureDistinctCall(self, arg): else: printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_deleteFeature(self, arg): '\n\tdeleteFeature {"feature": ""}\n' @@ -1349,7 +1508,7 @@ def do_deleteFeature(self, arg): else: if parmData['FEATURE'] in ('NAME',): - printWithNewLines('Can\'t delete feature %s!' % parmData['FEATURE'], 'B') + printWithNewLines('Can\'t delete feature %s!' % parmData['FEATURE'], 'B') return deleteCnt = 0 @@ -1417,7 +1576,6 @@ def do_deleteFeature(self, arg): else: printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_setFeature(self, arg): '\n\tsetFeature {"feature": "", "behavior": ""}' \ '\n\tsetFeature {"feature": "", "comparison": ""}\n' @@ -1466,7 +1624,7 @@ def do_setFeature(self, arg): printWithNewLines('Invalid behavior: %s' % parmData['BEHAVIOR']) elif parmCode == 'ANONYMIZE': - if parmData['ANONYMIZE'].upper() in ('YES', 'Y', 'NO','N'): + if parmData['ANONYMIZE'].upper() in ('YES', 'Y', 'NO', 'N'): self.cfgData['G2_CONFIG']['CFG_FTYPE'][listID]['ANONYMIZE'] = 'Yes' if parmData['ANONYMIZE'].upper() in ('YES', 'Y') else 'No' printWithNewLines('Anonymize setting updated!') self.configUpdated = True @@ -1474,7 +1632,7 @@ def do_setFeature(self, arg): printWithNewLines('Invalid anonymize setting: %s' % parmData['ANONYMIZE']) elif parmCode == 'CANDIDATES': - if parmData['CANDIDATES'].upper() in ('YES', 'Y', 'NO','N'): + if parmData['CANDIDATES'].upper() in ('YES', 'Y', 'NO', 'N'): self.cfgData['G2_CONFIG']['CFG_FTYPE'][listID]['USED_FOR_CAND'] = 'Yes' if parmData['CANDIDATES'].upper() in ('YES', 'Y') else 'No' printWithNewLines('Candidates setting updated!') self.configUpdated = True @@ -1537,7 +1695,6 @@ def do_setFeature(self, arg): print() - def do_addFeature(self, arg): '\n\taddFeature {"feature": "", "behavior": "", "elementList": [""]\n\n\tNote the [ and ]') @@ -1572,7 +1729,7 @@ def do_addFeature(self, arg): if 'ID' in parmData: ftypeID = int(parmData['ID']) else: - ftypeID = maxID + 1 if maxID >=1000 else 1000 + ftypeID = maxID + 1 if maxID >= 1000 else 1000 #--default for missing values parmData['ID'] = ftypeID @@ -1597,7 +1754,7 @@ def do_addFeature(self, arg): else: fclassID = fclassRecord['FCLASS_ID'] - sfuncID = 0 #--standardization function + sfuncID = 0 # --standardization function if 'STANDARDIZE' in parmData and len(parmData['STANDARDIZE']) != 0: parmData['STANDARDIZE'] = parmData['STANDARDIZE'].upper() sfuncRecord = self.getRecord('CFG_SFUNC', 'SFUNC_CODE', parmData['STANDARDIZE']) @@ -1607,7 +1764,7 @@ def do_addFeature(self, arg): printWithNewLines('Invalid standardization code: %s' % parmData['STANDARDIZE'], 'B') return - efuncID = 0 #--expression function + efuncID = 0 # --expression function if 'EXPRESSION' in parmData and len(parmData['EXPRESSION']) != 0: parmData['EXPRESSION'] = parmData['EXPRESSION'].upper() efuncRecord = self.getRecord('CFG_EFUNC', 'EFUNC_CODE', parmData['EXPRESSION']) @@ -1617,7 +1774,7 @@ def do_addFeature(self, arg): printWithNewLines('Invalid expression code: %s' % parmData['EXPRESSION'], 'B') return - cfuncID = 0 #--comparison function + cfuncID = 0 # --comparison function if 'COMPARISON' in parmData and len(parmData['COMPARISON']) != 0: parmData['COMPARISON'] = parmData['COMPARISON'].upper() cfuncRecord = self.getRecord('CFG_CFUNC', 'CFUNC_CODE', parmData['COMPARISON']) @@ -1683,7 +1840,7 @@ def do_addFeature(self, arg): #--add the distinct value call (not supported through here yet) dfcallID = 0 - dfuncID = 0 #--more efficent to leave it null + dfuncID = 0 # --more efficent to leave it null if dfuncID > 0: for i in range(len(self.cfgData['G2_CONFIG']['CFG_DFCALL'])): if self.cfgData['G2_CONFIG']['CFG_DFCALL'][i]['DFCALL_ID'] > dfcallID: @@ -1761,7 +1918,7 @@ def do_addFeature(self, arg): #--add if not found if felemID == 0: - felemID = maxID + 1 if maxID >=1000 else 1000 + felemID = maxID + 1 if maxID >= 1000 else 1000 newRecord = {} newRecord['FELEM_ID'] = felemID newRecord['FELEM_CODE'] = elementRecord['ELEMENT'] @@ -1827,7 +1984,6 @@ def do_addFeature(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def getFeatureJson(self, ftypeRecord): fclassRecord = self.getRecord('CFG_FCLASS', 'FCLASS_ID', ftypeRecord['FCLASS_ID']) @@ -1863,7 +2019,7 @@ def getFeatureJson(self, ftypeRecord): elementRecord['element'] = felemRecord['FELEM_CODE'] elementRecord['expressed'] = 'No' if not efcallRecord or not self.getRecord('CFG_EFBOM', ['EFCALL_ID', 'FTYPE_ID', 'FELEM_ID'], [efcallRecord['EFCALL_ID'], fbomRecord['FTYPE_ID'], fbomRecord['FELEM_ID']]) else 'Yes' elementRecord['compared'] = 'No' if not cfcallRecord or not self.getRecord('CFG_CFBOM', ['CFCALL_ID', 'FTYPE_ID', 'FELEM_ID'], [cfcallRecord['CFCALL_ID'], fbomRecord['FTYPE_ID'], fbomRecord['FELEM_ID']]) else 'Yes' - elementRecord['display'] = 'No' if fbomRecord['DISPLAY_LEVEL'] == 0 else 'Yes' + elementRecord['display'] = 'No' if fbomRecord['DISPLAY_LEVEL'] == 0 else 'Yes' elementList.append(elementRecord) else: elementList.append(felemRecord['FELEM_CODE']) @@ -1873,7 +2029,6 @@ def getFeatureJson(self, ftypeRecord): return jsonString - def do_addToNamehash(self, arg): '\n\taddToNamehash {"feature": "", "element": ""}\n' @@ -1888,7 +2043,7 @@ def do_addToNamehash(self, arg): try: nameHasher_efuncID = self.getRecord('CFG_EFUNC', 'EFUNC_CODE', 'NAME_HASHER')['EFUNC_ID'] nameHasher_efcallID = self.getRecord('CFG_EFCALL', 'EFUNC_ID', nameHasher_efuncID)['EFCALL_ID'] - except: + except Exception: nameHasher_efcallID = 0 if not nameHasher_efcallID: printWithNewLines('Name hasher function not found!', 'B') @@ -1917,7 +2072,7 @@ def do_addToNamehash(self, arg): if ftypeID != -1: if not self.getRecord('CFG_FBOM', ['FTYPE_ID', 'FELEM_ID'], [ftypeID, felemID]): - printWithNewLines('%s is not an element of feature %s'% (parmData['ELEMENT'], parmData['FEATURE']), 'B') + printWithNewLines('%s is not an element of feature %s' % (parmData['ELEMENT'], parmData['FEATURE']), 'B') return nameHasher_execOrder = 0 @@ -1942,7 +2097,6 @@ def do_addToNamehash(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def do_deleteFromNamehash(self, arg): '\n\tdeleteFromNamehash {"feature": "", "element": ""}\n' @@ -1957,7 +2111,7 @@ def do_deleteFromNamehash(self, arg): try: nameHasher_efuncID = self.getRecord('CFG_EFUNC', 'EFUNC_CODE', 'NAME_HASHER')['EFUNC_ID'] nameHasher_efcallID = self.getRecord('CFG_EFCALL', 'EFUNC_ID', nameHasher_efuncID)['EFCALL_ID'] - except: + except Exception: nameHasher_efcallID = 0 if not nameHasher_efcallID: printWithNewLines('Name hasher function not found!', 'B') @@ -1991,7 +2145,6 @@ def do_deleteFromNamehash(self, arg): printWithNewLines('Record not found!', 'B') printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_addToNameSSNLast4hash(self, arg): '\n\taddToNameSSNLast4hash {"feature": "", "element": ""}\n' @@ -2009,7 +2162,7 @@ def do_addToNameSSNLast4hash(self, arg): for i in range(len(self.cfgData['G2_CONFIG']['CFG_EFCALL'])-1, -1, -1): if self.cfgData['G2_CONFIG']['CFG_EFCALL'][i]['EFUNC_ID'] == ssnLast4Hasher_efuncID and self.cfgData['G2_CONFIG']['CFG_EFCALL'][i]['FTYPE_ID'] == self.getRecord('CFG_FTYPE', 'FTYPE_CODE', 'SSN_LAST4')['FTYPE_ID']: ssnLast4Hasher_efcallID = self.cfgData['G2_CONFIG']['CFG_EFCALL'][i]['EFCALL_ID'] - except: + except Exception: ssnLast4Hasher_efcallID = 0 if not ssnLast4Hasher_efcallID: printWithNewLines('SSNLast4 hasher function not found!', 'B') @@ -2038,7 +2191,7 @@ def do_addToNameSSNLast4hash(self, arg): if ftypeID != -1: if not self.getRecord('CFG_FBOM', ['FTYPE_ID', 'FELEM_ID'], [ftypeID, felemID]): - printWithNewLines('%s is not an element of feature %s'% (parmData['ELEMENT'], parmData['FEATURE']), 'B') + printWithNewLines('%s is not an element of feature %s' % (parmData['ELEMENT'], parmData['FEATURE']), 'B') return ssnLast4Hasher_execOrder = 0 @@ -2063,7 +2216,6 @@ def do_addToNameSSNLast4hash(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def do_deleteFromSSNLast4hash(self, arg): '\n\tdeleteFromSSNLast4hash {"feature": "", "element": ""}\n' @@ -2081,7 +2233,7 @@ def do_deleteFromSSNLast4hash(self, arg): for i in range(len(self.cfgData['G2_CONFIG']['CFG_EFCALL'])-1, -1, -1): if self.cfgData['G2_CONFIG']['CFG_EFCALL'][i]['EFUNC_ID'] == ssnLast4Hasher_efuncID and self.cfgData['G2_CONFIG']['CFG_EFCALL'][i]['FTYPE_ID'] == self.getRecord('CFG_FTYPE', 'FTYPE_CODE', 'SSN_LAST4')['FTYPE_ID']: ssnLast4Hasher_efcallID = self.cfgData['G2_CONFIG']['CFG_EFCALL'][i]['EFCALL_ID'] - except: + except Exception: ssnLast4Hasher_efcallID = 0 if not ssnLast4Hasher_efcallID: printWithNewLines('SSNLast4 hasher function not found!', 'B') @@ -2119,23 +2271,25 @@ def do_deleteFromSSNLast4hash(self, arg): # ===== attribute commands ===== def do_listAttributes(self, arg): - '\n\tlistAttributes\n' + '\n\tlistAttributes [search_value]\n' print() - for attrRecord in sorted(self.getRecordList('CFG_ATTR'), key = lambda k: k['ATTR_ID']): - print(self.getAttributeJson(attrRecord)) + for attrRecord in sorted(self.getRecordList('CFG_ATTR'), key=lambda k: k['ATTR_ID']): + if arg and arg.lower() not in str(attrRecord).lower(): + continue + printWithNewLines(self.getAttributeJson(attrRecord), 'E') print() - def do_listAttributeClasses(self, arg): - '\n\tlistAttributeClasses\n' + '\n\tlistAttributeClasses [search_value]\n' print() for attrClass in self.attributeClassList: + if arg and arg.lower() not in str(attrClass).lower(): + continue print('{"attributeClass": "%s"}' % attrClass) print() - def do_getAttribute(self, arg): '\n\tgetAttribute {"attribute": ""}' \ '\n\tgetAttribute {"feature": ""}\t\tList all the attributes for a feature\n' @@ -2164,11 +2318,8 @@ def do_getAttribute(self, arg): if not attrRecords: printWithNewLines('Record not found!', 'B') else: - print() - for attrRecord in sorted(attrRecords, key = lambda k: k['ATTR_ID']): - print(self.getAttributeJson(attrRecord)) - print() - + for attrRecord in sorted(attrRecords, key=lambda k: k['ATTR_ID']): + self.printJsonResponse(self.getAttributeJson(attrRecord)) def do_deleteAttribute(self, arg): '\n\tdeleteAttribute {"attribute": ""}' \ @@ -2204,7 +2355,6 @@ def do_deleteAttribute(self, arg): printWithNewLines('Record not found!', 'B') printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_addEntityScore(self, arg): '\n\taddEntityScore {"behavior": "", "grouperFeat": "", "richnessScore": "", "exclusivityScore": ""}\n' @@ -2237,7 +2387,6 @@ def do_addEntityScore(self, arg): if self.doDebug: debug(newRecord) - def do_addAttribute(self, arg): '\n\taddAttribute {"attribute": ""}' \ '\n\n\taddAttribute {"attribute": "", "class": "", "feature": "", "element": ""}' \ @@ -2281,7 +2430,7 @@ def do_addAttribute(self, arg): return else: if not self.getRecord('CFG_FBOM', ['FTYPE_ID', 'FELEM_ID'], [ftypeRecord['FTYPE_ID'], felemRecord['FELEM_ID']]): - printWithNewLines('%s is not an element of feature %s'% (parmData['ELEMENT'], parmData['FEATURE']), 'B') + printWithNewLines('%s is not an element of feature %s' % (parmData['ELEMENT'], parmData['FEATURE']), 'B') return else: parmData['ELEMENT'] = None @@ -2335,7 +2484,6 @@ def do_addAttribute(self, arg): if self.doDebug: debug(newRecord) - def getAttributeJson(self, attributeRecord): if 'ADVANCED' not in attributeRecord: @@ -2361,14 +2509,15 @@ def getAttributeJson(self, attributeRecord): # ===== element commands ===== def do_listElements(self, arg): - '\n\tlistElements\n' + '\n\tlistElements [search_value]\n' print() - for elemRecord in sorted(self.getRecordList('CFG_FELEM'), key = lambda k: k['FELEM_ID']): + for elemRecord in sorted(self.getRecordList('CFG_FELEM'), key=lambda k: k['FELEM_ID']): + if arg and arg.lower() not in str(elemRecord).lower(): + continue print('{"id": %i, "code": "%s", "tokenize": "%s", "datatype": "%s"}' % (elemRecord['FELEM_ID'], elemRecord['FELEM_CODE'], elemRecord['TOKENIZE'], elemRecord['DATA_TYPE'])) print() - def do_getElement(self, arg): '\n\tgetElement {"element": ""}\n' @@ -2386,8 +2535,7 @@ def do_getElement(self, arg): if not felemRecord: printWithNewLines('Element %s not found!' % parmData['ELEMENT'], 'B') else: - printWithNewLines('{"id": %s, "code": %s, "datatype": %s, "tokenize": %s}' % (felemRecord['FELEM_ID'], felemRecord['FELEM_CODE'], felemRecord['DATA_TYPE'], felemRecord['TOKENIZE']), 'B') - + self.printJsonResponse('{"id": %s, "code": "%s", "datatype": "%s", "tokenize": "%s"}' % (felemRecord['FELEM_ID'], felemRecord['FELEM_CODE'], felemRecord['DATA_TYPE'], felemRecord['TOKENIZE'])) def do_addStandardizeFunc(self, arg): '\n\taddStandardizeFunc {"function":"", "connectStr":""}' \ @@ -2426,7 +2574,7 @@ def do_addStandardizeFunc(self, arg): if 'ID' in parmData: sfuncID = int(parmData['ID']) else: - sfuncID = max(maxID) + 1 if max(maxID) >=1000 else 1000 + sfuncID = max(maxID) + 1 if max(maxID) >= 1000 else 1000 newRecord = {} newRecord['SFUNC_ID'] = sfuncID @@ -2441,7 +2589,6 @@ def do_addStandardizeFunc(self, arg): if self.doDebug: debug(newRecord) - def do_addStandardizeCall(self, arg): '\n\taddStandardizeCall {"element":"", "function":"", "execOrder":}' \ '\n\n\taddStandardizeCall {"element":"COUNTRY", "function":"STANDARDIZE_COUNTRY", "execOrder":100}\n' @@ -2455,33 +2602,33 @@ def do_addStandardizeCall(self, arg): argError(arg, e) else: - featureIsSpecified = False; + featureIsSpecified = False ftypeID = -1 - if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 : + if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0: parmData['FEATURE'] = parmData['FEATURE'].upper() ftypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', parmData['FEATURE']) if not ftypeRecord: printWithNewLines('Invalid feature: %s.' % parmData['FEATURE'], 'B') return - featureIsSpecified = True; + featureIsSpecified = True ftypeID = ftypeRecord['FTYPE_ID'] - elementIsSpecified = False; + elementIsSpecified = False felemID = -1 - if 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0 : + if 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0: parmData['ELEMENT'] = parmData['ELEMENT'].upper() felemRecord = self.getRecord('CFG_FELEM', 'FELEM_CODE', parmData['ELEMENT']) if not felemRecord: printWithNewLines('Invalid element: %s.' % parmData['ELEMENT'], 'B') return - elementIsSpecified = True; + elementIsSpecified = True felemID = felemRecord['FELEM_ID'] - if featureIsSpecified == False and elementIsSpecified == False : + if featureIsSpecified is False and elementIsSpecified is False: printWithNewLines('No feature or element specified.', 'B') return - if featureIsSpecified == True and elementIsSpecified == True : + if featureIsSpecified is True and elementIsSpecified is True: printWithNewLines('Both feature and element specified. Must only use one, not both.', 'B') return @@ -2501,14 +2648,14 @@ def do_addStandardizeCall(self, arg): return maxID = [] - for i in range(len(self.cfgData['G2_CONFIG']['CFG_SFCALL'])) : + for i in range(len(self.cfgData['G2_CONFIG']['CFG_SFCALL'])): maxID.append(self.cfgData['G2_CONFIG']['CFG_SFCALL'][i]['SFCALL_ID']) sfcallID = 0 if 'ID' in parmData: sfcallID = int(parmData['ID']) else: - sfcallID = max(maxID) + 1 if max(maxID) >=1000 else 1000 + sfcallID = max(maxID) + 1 if max(maxID) >= 1000 else 1000 newRecord = {} newRecord['SFCALL_ID'] = sfcallID @@ -2522,7 +2669,6 @@ def do_addStandardizeCall(self, arg): if self.doDebug: debug(newRecord) - def do_addExpressionFunc(self, arg): '\n\taddExpressionFunc {"function":"", "connectStr":""}' \ @@ -2553,14 +2699,14 @@ def do_addExpressionFunc(self, arg): return maxID = [] - for i in range(len(self.cfgData['G2_CONFIG']['CFG_EFUNC'])) : + for i in range(len(self.cfgData['G2_CONFIG']['CFG_EFUNC'])): maxID.append(self.cfgData['G2_CONFIG']['CFG_EFUNC'][i]['EFUNC_ID']) efuncID = 0 if 'ID' in parmData: efuncID = int(parmData['ID']) else: - efuncID = max(maxID) + 1 if max(maxID) >=1000 else 1000 + efuncID = max(maxID) + 1 if max(maxID) >= 1000 else 1000 newRecord = {} newRecord['EFUNC_ID'] = efuncID @@ -2575,7 +2721,6 @@ def do_addExpressionFunc(self, arg): if self.doDebug: debug(newRecord) - def do_updateFeatureVersion(self, arg): '\n\tupdateFeatureVersion {"feature":"", "version":}\n' @@ -2585,9 +2730,9 @@ def do_updateFeatureVersion(self, arg): try: parmData = dictKeysUpper(json.loads(arg)) - if not 'FEATURE' in parmData or len(parmData['FEATURE']) == 0: + if 'FEATURE' not in parmData or len(parmData['FEATURE']) == 0: raise ValueError('Feature name is required!') - if not 'VERSION' in parmData: + if 'VERSION' not in parmData: raise ValueError('Version is required!') parmData['FEATURE'] = parmData['FEATURE'].upper() except (ValueError, KeyError) as e: @@ -2605,7 +2750,6 @@ def do_updateFeatureVersion(self, arg): if self.doDebug: debug(ftypeRecord) - def do_updateAttributeAdvanced(self, arg): '\n\tupdateAttributeAdvanced {"attribute":"", "advanced":"Yes"}\n' @@ -2615,9 +2759,9 @@ def do_updateAttributeAdvanced(self, arg): try: parmData = dictKeysUpper(json.loads(arg)) - if not 'ATTRIBUTE' in parmData or len(parmData['ATTRIBUTE']) == 0: + if 'ATTRIBUTE' not in parmData or len(parmData['ATTRIBUTE']) == 0: raise ValueError('Attribute name is required!') - if not 'ADVANCED' in parmData: + if 'ADVANCED' not in parmData: raise ValueError('Advanced value is required!') except (ValueError, KeyError) as e: argError(arg, e) @@ -2634,7 +2778,6 @@ def do_updateAttributeAdvanced(self, arg): if self.doDebug: debug(attrRecord) - def do_updateExpressionFuncVersion(self, arg): '\n\tupdateExpressionFuncVersion {"function":"", "version":""}\n' @@ -2644,9 +2787,9 @@ def do_updateExpressionFuncVersion(self, arg): try: parmData = dictKeysUpper(json.loads(arg)) - if not 'FUNCTION' in parmData or len(parmData['FUNCTION']) == 0: + if 'FUNCTION' not in parmData or len(parmData['FUNCTION']) == 0: raise ValueError('Function is required!') - if not 'VERSION' in parmData or len(parmData['VERSION']) == 0: + if 'VERSION' not in parmData or len(parmData['VERSION']) == 0: raise ValueError('Version is required!') parmData['FUNCTION'] = parmData['FUNCTION'].upper() except (ValueError, KeyError) as e: @@ -2661,10 +2804,10 @@ def do_updateExpressionFuncVersion(self, arg): funcRecord['FUNC_VER'] = parmData['VERSION'] self.configUpdated = True printWithNewLines('Successfully updated!', 'B') + if self.doDebug: debug(funcRecord) - def do_addComparisonFuncReturnCode(self, arg): '\n\taddComparisonFuncReturnCode {"function":"", "scoreName":""}' \ @@ -2702,7 +2845,7 @@ def do_addComparisonFuncReturnCode(self, arg): if 'ID' in parmData: cfrtnID = int(parmData['ID']) else: - cfrtnID = max(maxID) + 1 if max(maxID) >=1000 else 1000 + cfrtnID = max(maxID) + 1 if max(maxID) >= 1000 else 1000 execOrder = 0 for i in range(len(self.cfgData['G2_CONFIG']['CFG_CFRTN'])): @@ -2727,7 +2870,6 @@ def do_addComparisonFuncReturnCode(self, arg): if self.doDebug: debug(newRecord) - def do_addComparisonFunc(self, arg): '\n\taddComparisonFunc {"function":"", "connectStr":""}' \ @@ -2759,14 +2901,14 @@ def do_addComparisonFunc(self, arg): return maxID = [] - for i in range(len(self.cfgData['G2_CONFIG']['CFG_CFUNC'])) : + for i in range(len(self.cfgData['G2_CONFIG']['CFG_CFUNC'])): maxID.append(self.cfgData['G2_CONFIG']['CFG_CFUNC'][i]['CFUNC_ID']) cfuncID = 0 if 'ID' in parmData: cfuncID = int(parmData['ID']) else: - cfuncID = max(maxID) + 1 if max(maxID) >=1000 else 1000 + cfuncID = max(maxID) + 1 if max(maxID) >= 1000 else 1000 newRecord = {} newRecord['CFUNC_ID'] = cfuncID @@ -2782,8 +2924,6 @@ def do_addComparisonFunc(self, arg): if self.doDebug: debug(newRecord) - - def do_addExpressionCall(self, arg): '\n\taddExpressionCall {"element":"", "function":"", "execOrder":, expressionFeature":, "virtual":"No","elementList": [""]\n\n\tNote the [ and ]') @@ -2803,33 +2943,33 @@ def do_addExpressionCall(self, arg): argError(arg, e) else: - featureIsSpecified = False; + featureIsSpecified = False ftypeID = -1 - if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 : + if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0: parmData['FEATURE'] = parmData['FEATURE'].upper() ftypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', parmData['FEATURE']) if not ftypeRecord: printWithNewLines('Invalid feature: %s.' % parmData['FEATURE'], 'B') return - featureIsSpecified = True; + featureIsSpecified = True ftypeID = ftypeRecord['FTYPE_ID'] - elementIsSpecified = False; + elementIsSpecified = False felemID = -1 - if 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0 : + if 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0: parmData['ELEMENT'] = parmData['ELEMENT'].upper() felemRecord = self.getRecord('CFG_FELEM', 'FELEM_CODE', parmData['ELEMENT']) if not felemRecord: printWithNewLines('Invalid element: %s.' % parmData['ELEMENT'], 'B') return - elementIsSpecified = True; + elementIsSpecified = True felemID = felemRecord['FELEM_ID'] - if featureIsSpecified == False and elementIsSpecified == False : + if featureIsSpecified is False and elementIsSpecified is False: printWithNewLines('No feature or element specified.', 'B') return - if featureIsSpecified == True and elementIsSpecified == True : + if featureIsSpecified is True and elementIsSpecified is True: printWithNewLines('Both feature and element specified. Must only use one, not both.', 'B') return @@ -2867,12 +3007,12 @@ def do_addExpressionCall(self, arg): if 'ID' in parmData: efcallID = int(parmData['ID']) else: - efcallID = max(maxID) + 1 if max(maxID) >=1000 else 1000 + efcallID = max(maxID) + 1 if max(maxID) >= 1000 else 1000 isVirtual = parmData['VIRTUAL'] if 'VIRTUAL' in parmData else 'No' efeatFTypeID = -1 - if 'EXPRESSIONFEATURE' in parmData and len(parmData['EXPRESSIONFEATURE']) != 0 : + if 'EXPRESSIONFEATURE' in parmData and len(parmData['EXPRESSIONFEATURE']) != 0: parmData['EXPRESSIONFEATURE'] = parmData['EXPRESSIONFEATURE'].upper() expressionFTypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', parmData['EXPRESSIONFEATURE']) if not expressionFTypeRecord: @@ -2886,42 +3026,42 @@ def do_addExpressionCall(self, arg): elementCount += 1 elementRecord = dictKeysUpper(element) - bomFTypeIsSpecified = False; + bomFTypeIsSpecified = False bomFTypeID = -1 - if 'FEATURE' in elementRecord and len(elementRecord['FEATURE']) != 0 : + if 'FEATURE' in elementRecord and len(elementRecord['FEATURE']) != 0: elementRecord['FEATURE'] = elementRecord['FEATURE'].upper() bomFTypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', elementRecord['FEATURE']) if not bomFTypeRecord: printWithNewLines('Invalid BOM feature: %s.' % elementRecord['FEATURE'], 'B') return - bomFTypeIsSpecified = True; + bomFTypeIsSpecified = True bomFTypeID = bomFTypeRecord['FTYPE_ID'] - bomFElemIsSpecified = False; + bomFElemIsSpecified = False bomFElemID = -1 - if 'ELEMENT' in elementRecord and len(elementRecord['ELEMENT']) != 0 : + if 'ELEMENT' in elementRecord and len(elementRecord['ELEMENT']) != 0: elementRecord['ELEMENT'] = elementRecord['ELEMENT'].upper() bomFElemRecord = self.getRecord('CFG_FELEM', 'FELEM_CODE', elementRecord['ELEMENT']) if not bomFElemRecord: printWithNewLines('Invalid BOM element: %s.' % elementRecord['ELEMENT'], 'B') return - bomFElemIsSpecified = True; + bomFElemIsSpecified = True bomFElemID = bomFElemRecord['FELEM_ID'] - if bomFElemIsSpecified == False : + if bomFElemIsSpecified is False: printWithNewLines('No BOM element specified on BOM entry.', 'B') return - bomFTypeFeatureLinkIsSpecified = False; - if 'FEATURELINK' in elementRecord and len(elementRecord['FEATURELINK']) != 0 : + bomFTypeFeatureLinkIsSpecified = False + if 'FEATURELINK' in elementRecord and len(elementRecord['FEATURELINK']) != 0: elementRecord['FEATURELINK'] = elementRecord['FEATURELINK'].upper() if elementRecord['FEATURELINK'] != 'PARENT': printWithNewLines('Invalid feature link value: %s. (Must use \'parent\')' % elementRecord['FEATURELINK'], 'B') return - bomFTypeFeatureLinkIsSpecified = True; + bomFTypeFeatureLinkIsSpecified = True bomFTypeID = 0 - if bomFTypeIsSpecified == True and bomFTypeFeatureLinkIsSpecified == True : + if bomFTypeIsSpecified is True and bomFTypeFeatureLinkIsSpecified is True: printWithNewLines('Cannot specify both ftype and feature-link on single function BOM entry.', 'B') return @@ -2949,16 +3089,16 @@ def do_addExpressionCall(self, arg): elementRecord = dictKeysUpper(element) bomFTypeID = -1 - if 'FEATURE' in elementRecord and len(elementRecord['FEATURE']) != 0 : + if 'FEATURE' in elementRecord and len(elementRecord['FEATURE']) != 0: bomFTypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', elementRecord['FEATURE']) bomFTypeID = bomFTypeRecord['FTYPE_ID'] bomFElemID = -1 - if 'ELEMENT' in elementRecord and len(elementRecord['ELEMENT']) != 0 : + if 'ELEMENT' in elementRecord and len(elementRecord['ELEMENT']) != 0: bomFElemRecord = self.getRecord('CFG_FELEM', 'FELEM_CODE', elementRecord['ELEMENT']) bomFElemID = bomFElemRecord['FELEM_ID'] - if 'FEATURELINK' in elementRecord and len(elementRecord['FEATURELINK']) != 0 : + if 'FEATURELINK' in elementRecord and len(elementRecord['FEATURELINK']) != 0: elementRecord['FEATURELINK'] = elementRecord['FEATURELINK'].upper() bomFTypeID = 0 @@ -2978,7 +3118,6 @@ def do_addExpressionCall(self, arg): self.configUpdated = True printWithNewLines('Successfully added!', 'B') - def do_deleteExpressionCall(self, arg): '\n\tdeleteExpressionCall {"id": ""}\n' @@ -3012,7 +3151,6 @@ def do_deleteExpressionCall(self, arg): if self.cfgData['G2_CONFIG']['CFG_EFBOM'][i][searchField] == searchValue: del self.cfgData['G2_CONFIG']['CFG_EFBOM'][i] - def do_addElement(self, arg): '\n\taddElement {"element": ""}' \ @@ -3051,13 +3189,13 @@ def do_addElement(self, arg): return maxID = [] - for i in range(len(self.cfgData['G2_CONFIG']['CFG_FELEM'])) : + for i in range(len(self.cfgData['G2_CONFIG']['CFG_FELEM'])): maxID.append(self.cfgData['G2_CONFIG']['CFG_FELEM'][i]['FELEM_ID']) if 'ID' in parmData: felemID = int(parmData['ID']) else: - felemID = max(maxID) + 1 if max(maxID) >=1000 else 1000 + felemID = max(maxID) + 1 if max(maxID) >= 1000 else 1000 newRecord = {} newRecord['FELEM_ID'] = felemID @@ -3071,7 +3209,6 @@ def do_addElement(self, arg): if self.doDebug: debug(newRecord) - def do_addElementToFeature(self, arg): '\n\taddElementToFeature {"feature": "", "element": ""}' \ @@ -3152,7 +3289,7 @@ def do_addElementToFeature(self, arg): if ( ( parmData['DATATYPE'] and len(parmData['DATATYPE'].strip()) > 0 and parmData['DATATYPE'] != felemRecord['DATA_TYPE'] ) or ( parmData['TOKENIZE'] and len(parmData['TOKENIZE'].strip()) > 0 and parmData['TOKENIZE'] != felemRecord['TOKENIZE'] ) - ) : + ): printWithNewLines('Element %s already exists with conflicting parameters, check with listElement %s' % (parmData['ELEMENT'], parmData['ELEMENT']), 'B') return else: @@ -3169,7 +3306,7 @@ def do_addElementToFeature(self, arg): if 'ID' in parmData: felemID = int(parmData['ID']) else: - felemID = maxID + 1 if maxID >=1000 else 1000 + felemID = maxID + 1 if maxID >= 1000 else 1000 newRecord = {} newRecord['FELEM_ID'] = felemID @@ -3209,7 +3346,6 @@ def do_addElementToFeature(self, arg): if self.doDebug: debug(newRecord) - def do_setFeatureElementDisplayLevel(self, arg): '\n\tsetFeatureElementDisplayLevel {"feature": "", "element": "", "display_level": }\n' @@ -3223,7 +3359,7 @@ def do_setFeatureElementDisplayLevel(self, arg): print('\nError with argument(s) or parsing JSON - %s \n' % e) else: - if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 and 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0 : + if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 and 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0: parmData['FEATURE'] = parmData['FEATURE'].upper() ftypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', parmData['FEATURE']) @@ -3241,8 +3377,7 @@ def do_setFeatureElementDisplayLevel(self, arg): printWithNewLines('Both a feature and element must be specified!', 'B') return - - if 'DISPLAY_LEVEL' in parmData : + if 'DISPLAY_LEVEL' in parmData: displayLevel = int(parmData['DISPLAY_LEVEL']) else: printWithNewLines('Display level must be specified!', 'B') @@ -3257,7 +3392,6 @@ def do_setFeatureElementDisplayLevel(self, arg): if self.doDebug: debug(self.cfgData['G2_CONFIG']['CFG_FBOM'][i]) - def do_setFeatureElementDerived(self, arg): '\n\tsetFeatureElementDerived {"feature": "", "element": "", "derived": }\n' @@ -3271,14 +3405,14 @@ def do_setFeatureElementDerived(self, arg): print('\nError with argument(s) or parsing JSON - %s \n' % e) else: - if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 and 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0 : + if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 and 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0: parmData['FEATURE'] = parmData['FEATURE'].upper() ftypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', parmData['FEATURE']) if not ftypeRecord: printWithNewLines('Invalid feature: %s. Use listFeatures to see valid features.' % parmData['FEATURE'], 'B') return - + parmData['ELEMENT'] = parmData['ELEMENT'].upper() felemRecord = self.getRecord('CFG_FELEM', 'FELEM_CODE', parmData['ELEMENT']) if not felemRecord: @@ -3288,8 +3422,8 @@ def do_setFeatureElementDerived(self, arg): else: printWithNewLines('Both a feature and element must be specified!', 'B') return - - if 'DERIVED' in parmData : + + if 'DERIVED' in parmData: derived = parmData['DERIVED'] else: printWithNewLines('Derived status must be specified!', 'B') @@ -3304,7 +3438,6 @@ def do_setFeatureElementDerived(self, arg): if self.doDebug: debug(self.cfgData['G2_CONFIG']['CFG_FBOM'][i]) - def do_deleteElementFromFeature(self, arg): '\n\tdeleteElementFromFeature {"feature": "", "element": ""}\n' @@ -3317,7 +3450,7 @@ def do_deleteElementFromFeature(self, arg): print('\nError with argument(s) or parsing JSON - %s \n' % e) else: - if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 and 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0 : + if 'FEATURE' in parmData and len(parmData['FEATURE']) != 0 and 'ELEMENT' in parmData and len(parmData['ELEMENT']) != 0: parmData['FEATURE'] = parmData['FEATURE'].upper() ftypeRecord = self.getRecord('CFG_FTYPE', 'FTYPE_CODE', parmData['FEATURE']) @@ -3336,7 +3469,7 @@ def do_deleteElementFromFeature(self, arg): return deleteCnt = 0 - for i in range(len(self.cfgData['G2_CONFIG']['CFG_FBOM'])-1,-1,-1): + for i in range(len(self.cfgData['G2_CONFIG']['CFG_FBOM'])-1, -1, -1): if int(self.cfgData['G2_CONFIG']['CFG_FBOM'][i]['FTYPE_ID']) == ftypeRecord['FTYPE_ID'] and int(self.cfgData['G2_CONFIG']['CFG_FBOM'][i]['FELEM_ID']) == felemRecord['FELEM_ID'] : del self.cfgData['G2_CONFIG']['CFG_FBOM'][i] deleteCnt = 1 @@ -3347,7 +3480,6 @@ def do_deleteElementFromFeature(self, arg): else: printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_deleteElement(self, arg): '\n\tdeleteElement {"feature": "", "element": ""}\n' @@ -3367,32 +3499,32 @@ def do_deleteElement(self, arg): usedIn = [] for i in range(len(self.cfgData['G2_CONFIG']['CFG_FBOM'])): - if int(self.cfgData['G2_CONFIG']['CFG_FBOM'][i]['FELEM_ID']) == felemRecord['FELEM_ID'] : + if int(self.cfgData['G2_CONFIG']['CFG_FBOM'][i]['FELEM_ID']) == felemRecord['FELEM_ID']: for j in range(len(self.cfgData['G2_CONFIG']['CFG_FTYPE'])): - if int(self.cfgData['G2_CONFIG']['CFG_FTYPE'][j]['FTYPE_ID']) == self.cfgData['G2_CONFIG']['CFG_FBOM'][i]['FTYPE_ID'] : + if int(self.cfgData['G2_CONFIG']['CFG_FTYPE'][j]['FTYPE_ID']) == self.cfgData['G2_CONFIG']['CFG_FBOM'][i]['FTYPE_ID']: usedIn.append(self.cfgData['G2_CONFIG']['CFG_FTYPE'][j]['FTYPE_CODE']) if usedIn: - printWithNewLines('Can\'t delete %s, it is used in these feature(s): %s' % (parmData['ELEMENT'], usedIn) ,'B') + printWithNewLines('Can\'t delete %s, it is used in these feature(s): %s' % (parmData['ELEMENT'], usedIn), 'B') return else: deleteCnt = 0 for i in range(len(self.cfgData['G2_CONFIG']['CFG_FELEM'])): - if int(self.cfgData['G2_CONFIG']['CFG_FELEM'][i]['FELEM_ID']) == felemRecord['FELEM_ID'] : + if int(self.cfgData['G2_CONFIG']['CFG_FELEM'][i]['FELEM_ID']) == felemRecord['FELEM_ID']: del self.cfgData['G2_CONFIG']['CFG_FELEM'][i] deleteCnt = 1 self.configUpdated = True + break if deleteCnt == 0: printWithNewLines('Record not found!', 'B') else: printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_listExpressionCalls(self, arg): - '\n\tVerifies expression call configurations\n' + '\n\tlistExpressionCalls [search_value]\n' efcallList = [] - for efcallRecord in sorted(self.cfgData['G2_CONFIG']['CFG_EFCALL'], key = lambda k: (k['FTYPE_ID'], k['EXEC_ORDER'])): + for efcallRecord in sorted(self.cfgData['G2_CONFIG']['CFG_EFCALL'], key=lambda k: (k['FTYPE_ID'], k['EXEC_ORDER'])): efuncRecord = self.getRecord('CFG_EFUNC', 'EFUNC_ID', efcallRecord['EFUNC_ID']) ftypeRecord1 = self.getRecord('CFG_FTYPE', 'FTYPE_ID', efcallRecord['FTYPE_ID']) ftypeRecord2 = self.getRecord('CFG_FTYPE', 'FTYPE_ID', efcallRecord['EFEAT_FTYPE_ID']) @@ -3430,10 +3562,12 @@ def do_listExpressionCalls(self, arg): efcallList.append(efcallDict) - + print() for efcallDict in efcallList: - print(json.dumps(efcallDict)) - + if arg and arg.lower() not in str(efcallDict).lower(): + continue + printWithNewLines(json.dumps(efcallDict), 'E') + print() # ===== misc commands ===== @@ -3483,12 +3617,15 @@ def do_setDistinct(self, arg): return def do_listGenericThresholds(self, arg): - '\n\tlistGenericThresholds\n' + '\n\tlistGenericThresholds [search_value]\n' + planCode = {} planCode[1] = 'load' planCode[2] = 'search' print() - for thisRecord in sorted(self.getRecordList('CFG_GENERIC_THRESHOLD'), key = lambda k: k['GPLAN_ID']): + for thisRecord in sorted(self.getRecordList('CFG_GENERIC_THRESHOLD'), key=lambda k: k['GPLAN_ID']): + if arg and arg.lower() not in str(thisRecord).lower(): + continue print('{"plan": "%s", "behavior": "%s", "candidateCap": %s, "scoringCap": %s}' % (planCode[thisRecord['GPLAN_ID']], thisRecord['BEHAVIOR'], thisRecord['CANDIDATE_CAP'], thisRecord['SCORING_CAP'])) #{ # "BEHAVIOR": "NAME", @@ -3549,50 +3686,48 @@ def do_templateAdd(self, arg): '\n\ttemplateAdd {"feature": "customer_number", "template": "global_id", "behavior": "F1E", "comparison": "exact_comp"}' \ '\n\nType "templateAdd List" to get a list of valid templates.\n' - validTemplates = {} - validTemplates['GLOBAL_ID'] = {'DESCRIPTION': 'globally unique identifer (like an ssn, a credit card, or a medicare_id)', - 'BEHAVIOR': ['F1', 'F1E', 'F1ES'], + validTemplates['GLOBAL_ID'] = {'DESCRIPTION': 'globally unique identifer (like an ssn, a credit card, or a medicare_id)', + 'BEHAVIOR': ['F1', 'F1E', 'F1ES'], 'CANDIDATES': ['No'], 'STANDARDIZE': ['PARSE_ID'], 'EXPRESSION': ['EXPRESS_ID'], - 'COMPARISON': ['ID_COMP', 'EXACT_COMP'], + 'COMPARISON': ['ID_COMP', 'EXACT_COMP'], 'FEATURE_CLASS': 'ISSUED_ID', 'ATTRIBUTE_CLASS': 'IDENTIFIER', - 'ELEMENTS': [{'element': 'ID_NUM', 'expressed': 'No', 'compared': 'no', 'display': 'Yes'}, - {'element': 'ID_NUM_STD', 'expressed': 'Yes', 'compared': 'yes', 'display': 'No'}], + 'ELEMENTS': [{'element': 'ID_NUM', 'expressed': 'No', 'compared': 'no', 'display': 'Yes'}, + {'element': 'ID_NUM_STD', 'expressed': 'Yes', 'compared': 'yes', 'display': 'No'}], 'ATTRIBUTES': [{'attribute': '', 'element': 'ID_NUM', 'required': 'Yes'}]} validTemplates['STATE_ID'] = {'DESCRIPTION': 'state issued identifier (like a drivers license)', - 'BEHAVIOR': ['F1', 'F1E', 'F1ES'], + 'BEHAVIOR': ['F1', 'F1E', 'F1ES'], 'CANDIDATES': ['No'], 'STANDARDIZE': ['PARSE_ID'], 'EXPRESSION': ['EXPRESS_ID'], - 'COMPARISON': ['ID_COMP'], + 'COMPARISON': ['ID_COMP'], 'FEATURE_CLASS': 'ISSUED_ID', 'ATTRIBUTE_CLASS': 'IDENTIFIER', - 'ELEMENTS': [{'element': 'ID_NUM', 'expressed': 'No', 'compared': 'no', 'display': 'Yes'}, - {'element': 'STATE', 'expressed': 'No', 'compared': 'yes', 'display': 'Yes'}, - {'element': 'ID_NUM_STD', 'expressed': 'Yes', 'compared': 'yes', 'display': 'No'}], + 'ELEMENTS': [{'element': 'ID_NUM', 'expressed': 'No', 'compared': 'no', 'display': 'Yes'}, + {'element': 'STATE', 'expressed': 'No', 'compared': 'yes', 'display': 'Yes'}, + {'element': 'ID_NUM_STD', 'expressed': 'Yes', 'compared': 'yes', 'display': 'No'}], 'ATTRIBUTES': [{'attribute': '_NUMBER', 'element': 'ID_NUM', 'required': 'Yes'}, {'attribute': '_STATE', 'element': 'STATE', 'required': 'No'}]} validTemplates['COUNTRY_ID'] = {'DESCRIPTION': 'country issued identifier (like a passport)', - 'BEHAVIOR': ['F1', 'F1E', 'F1ES'], + 'BEHAVIOR': ['F1', 'F1E', 'F1ES'], 'CANDIDATES': ['No'], 'STANDARDIZE': ['PARSE_ID'], 'EXPRESSION': ['EXPRESS_ID'], - 'COMPARISON': ['ID_COMP'], + 'COMPARISON': ['ID_COMP'], 'FEATURE_CLASS': 'ISSUED_ID', 'ATTRIBUTE_CLASS': 'IDENTIFIER', - 'ELEMENTS': [{'element': 'ID_NUM', 'expressed': 'No', 'compared': 'no', 'display': 'Yes'}, - {'element': 'COUNTRY', 'expressed': 'No', 'compared': 'yes', 'display': 'Yes'}, - {'element': 'ID_NUM_STD', 'expressed': 'Yes', 'compared': 'yes', 'display': 'No'}], + 'ELEMENTS': [{'element': 'ID_NUM', 'expressed': 'No', 'compared': 'no', 'display': 'Yes'}, + {'element': 'COUNTRY', 'expressed': 'No', 'compared': 'yes', 'display': 'Yes'}, + {'element': 'ID_NUM_STD', 'expressed': 'Yes', 'compared': 'yes', 'display': 'No'}], 'ATTRIBUTES': [{'attribute': '_NUMBER', 'element': 'ID_NUM', 'required': 'Yes'}, {'attribute': '_COUNTRY', 'element': 'COUNTRY', 'required': 'No'}]} - if arg and arg.upper() == 'LIST': print() for template in validTemplates: @@ -3600,7 +3735,7 @@ def do_templateAdd(self, arg): print('\t\tbehaviors:', validTemplates[template]['BEHAVIOR']) print('\t\tcomparisons:', validTemplates[template]['COMPARISON']) print() - return + return if not argCheck('templateAdd', arg, self.do_templateAdd.__doc__): return @@ -3702,9 +3837,9 @@ def do_templateAdd(self, arg): attributeData = {'attribute': attributeDict['attribute'].upper(), 'class': attributeClass, - 'feature': feature, - 'element': attributeDict['element'].upper(), - 'required': attributeDict['required']} + 'feature': feature, + 'element': attributeDict['element'].upper(), + 'required': attributeDict['required']} attributeParm = json.dumps(attributeData) printWithNewLines('addAttribute %s' % attributeParm, 'S') @@ -3724,16 +3859,16 @@ def getFragmentJson(self, record): f'"depends": "{record["ERFRAG_DEPENDS"]}"' \ f'}}' - def do_listFragments(self, arg): - '\n\tlistFragments\n' + '\n\tlistFragments [search_value]\n' print() - for thisRecord in sorted(self.getRecordList('CFG_ERFRAG'), key = lambda k: k['ERFRAG_ID']): - print(self.getFragmentJson(thisRecord)) + for thisRecord in sorted(self.getRecordList('CFG_ERFRAG'), key=lambda k: k['ERFRAG_ID']): + if arg and arg.lower() not in str(thisRecord).lower(): + continue + printWithNewLines(self.getFragmentJson(thisRecord), 'E') print() - def do_getFragment(self, arg): '\n\tgetFragment {"id": ""}' \ '\n\tgetFragment {"fragment": ""}\n' @@ -3765,11 +3900,10 @@ def do_getFragment(self, arg): printWithNewLines('Record not found!', 'B') else: print() - for thisRecord in sorted(foundRecords, key = lambda k: k['ERFRAG_ID']): - print(self.getFragmentJson(thisRecord)) + for thisRecord in sorted(foundRecords, key=lambda k: k['ERFRAG_ID']): + self.printJsonResponse(self.getFragmentJson(thisRecord)) print() - def do_deleteFragment(self, arg): '\n\tdeleteFragment {"id": ""}' \ '\n\tdeleteFragment {"fragment": ""}\n' @@ -3806,7 +3940,6 @@ def do_deleteFragment(self, arg): printWithNewLines('Record not found!', 'B') printWithNewLines('%s rows deleted!' % deleteCnt, 'B') - def do_setFragment(self, arg): '\n\tsetFragment {"id": "", "fragment": "", "source": ""}\n' @@ -3886,7 +4019,6 @@ def do_setFragment(self, arg): print() - def do_addFragment(self, arg): '\n\taddFragment {"id": "", "fragment": "", "source": ""}' \ '\n\n\tFor additional example structures, use getFragment or listFragments\n' @@ -3917,7 +4049,7 @@ def do_addFragment(self, arg): #--must have a source field if 'SOURCE' not in parmData: - printWithNewLines( 'A fragment source field is required!', 'B') + printWithNewLines('A fragment source field is required!', 'B') return #--compute dependencies from source @@ -3966,30 +4098,31 @@ def do_addFragment(self, arg): # ===== rule commands ===== - # ----------------------------- def getRuleJson(self, record): - return f'{{' \ - f'"id": "{record["ERRULE_ID"]}", ' \ - f'"rule": "{record["ERRULE_CODE"]}", ' \ - f'"tier": "{showNullableJsonNumeric(record["ERRULE_TIER"])}", ' \ - f'"resolve": "{record["RESOLVE"]}", ' \ - f'"relate": "{record["RELATE"]}", ' \ - f'"ref_score": "{record["REF_SCORE"]}", ' \ - f'"fragment": "{record["QUAL_ERFRAG_CODE"]}", ' \ - f'"disqualifier": "{showNullableJsonNumeric(record["DISQ_ERFRAG_CODE"])}", ' \ - f'"rtype_id": "{showNullableJsonNumeric(record["RTYPE_ID"])}"' \ - f'}}' - + return ( + f'{{' + f'"id": {record["ERRULE_ID"]}, ' + f'"rule": "{record["ERRULE_CODE"]}", ' + f'"desc": "{record["ERRULE_DESC"]}", ' + f'"resolve": "{record["RESOLVE"]}", ' + f'"relate": "{record["RELATE"]}", ' + f'"ref_score": {record["REF_SCORE"]}, ' + f'"fragment": "{record["QUAL_ERFRAG_CODE"]}", ' + f'"disqualifier": "{showNullableJsonNumeric(record["DISQ_ERFRAG_CODE"])}", ' + f'"rtype_id": {showNullableJsonNumeric(record["RTYPE_ID"])}, ' + f'"tier": {showNullableJsonNumeric(record["ERRULE_TIER"])}' + f'}}' + ) def do_listRules(self, arg): - '\n\tlistRules\n' + '\n\tlistRules [search_value]\n' print() - for thisRecord in sorted(self.getRecordList('CFG_ERRULE'), key = lambda k: k['ERRULE_ID']): - print(self.getRuleJson(thisRecord)) - print() - + for ruleRecord in sorted(self.getRecordList('CFG_ERRULE'), key=lambda k: k['ERRULE_ID']): + if arg and arg.lower() not in str(ruleRecord).lower(): + continue + printWithNewLines(self.getRuleJson(ruleRecord), 'E') def do_getRule(self, arg): '\n\tgetRule {"id": ""}\n' @@ -4004,6 +4137,7 @@ def do_getRule(self, arg): parmData = {"ID": arg} else: parmData = {"RULE": arg} + if 'RULE' in parmData and len(parmData['RULE'].strip()) != 0: searchField = 'ERRULE_CODE' searchValue = parmData['RULE'].upper() @@ -4021,11 +4155,10 @@ def do_getRule(self, arg): printWithNewLines('Record not found!', 'B') else: print() - for thisRecord in sorted(foundRecords, key = lambda k: k['ERRULE_ID']): - print(self.getRuleJson(thisRecord)) + for thisRecord in sorted(foundRecords, key=lambda k: k['ERRULE_ID']): + self.printJsonResponse(self.getRuleJson(thisRecord)) print() - def do_deleteRule(self, arg): '\n\tdeleteRule {"id": ""}\n' @@ -4039,9 +4172,10 @@ def do_deleteRule(self, arg): parmData = {"ID": arg} else: parmData = {"RULE": arg} + if 'RULE' in parmData and len(parmData['RULE'].strip()) != 0: searchField = 'ERRULE_CODE' - searchValue = parmData['FRAGMENT'].upper() + searchValue = arg.upper() elif 'ID' in parmData and int(parmData['ID']) != 0: searchField = 'ERRULE_ID' searchValue = int(parmData['ID']) @@ -4058,18 +4192,27 @@ def do_deleteRule(self, arg): deleteCnt += 1 self.configUpdated = True if deleteCnt == 0: - printWithNewLines('Record not found!', 'B') + printWithNewLines('Record not found!', 'S') printWithNewLines('%s rows deleted!' % deleteCnt, 'B') def do_setRule(self, arg): - '\n\tsetRule {"id": "", "rule": "", "desc": "", "fragment": "", "disqualifier": ""}\n' + '\n\tsetRule {"id": , "rule": "", "desc": "", "fragment": "", "disqualifier": ""}\n' if not argCheck('setRule', arg, self.do_setRule.__doc__): return try: parmData = dictKeysUpper(json.loads(arg)) + if not isinstance(parmData['ID'], int): + parmData['ID'] = int(parmData['ID']) + if parmData['RESOLVE'] and parmData['RESOLVE'].upper() not in ('YES', 'NO'): + printWithNewLines('ERROR: Resolve should be Yes or No', 'B') + return + if parmData['RELATE'] and parmData['RELATE'].upper() not in ('YES', 'NO'): + printWithNewLines('ERROR: Relate should be Yes or No', 'B') + return + except (ValueError, KeyError) as e: argError(arg, e) else: @@ -4109,8 +4252,17 @@ def do_setRule(self, arg): printWithNewLines('Rule disqualifier updated!') self.configUpdated = True - print() + elif parmCode == 'RESOLVE': + self.cfgData['G2_CONFIG']['CFG_ERRULE'][listID]['RESOLVE'] = parmData['RESOLVE'].capitalize() + printWithNewLines('Rule resolve updated!') + self.configUpdated = True + + elif parmCode == 'RELATE': + self.cfgData['G2_CONFIG']['CFG_ERRULE'][listID]['RELATE'] = parmData['RELATE'].capitalize() + printWithNewLines('Rule relate updated!') + self.configUpdated = True + print() def do_addRule(self, arg): '\n\taddRule {"id": 130, "rule": "SF1_CNAME", "tier": 30, "resolve": "Yes", "relate": "No", "ref_score": 8, "fragment": "SF1_CNAME", "disqualifier": "DIFF_EXCL", "rtype_id": 1}' \ @@ -4142,7 +4294,7 @@ def do_addRule(self, arg): #--must have a valid fragment field if 'FRAGMENT' not in parmData: - printWithNewLines( 'A fragment source field is required!', 'B') + printWithNewLines('A fragment source field is required!', 'B') return else: #--lookup the fragment code @@ -4165,18 +4317,27 @@ def do_addRule(self, arg): printWithNewLines('Invalid disqualifer reference: %s' % parmData['DISQUALIFIER'], 'B') return + if 'TIER' not in parmData or not parmData['TIER']: + parmData['TIER'] = None + else: + if not isinstance(parmData['TIER'], int) and parmData['TIER'].lower() != 'null': + try: + parmData['TIER'] = int(parmData['TIER']) + except ValueError: + printWithNewLines(f'Invalid TIER value ({parmData["TIER"]}), should be an integer', 'B') + return + newRecord = {} newRecord['ERRULE_ID'] = int(parmData['ID']) newRecord['ERRULE_CODE'] = parmData['RULE'] - newRecord['ERRULE_DESC'] = parmData['RULE'] + newRecord['ERRULE_DESC'] = parmData['DESC'] if 'DESC' in parmData and parmData['DESC'] else parmData['RULE'] newRecord['RESOLVE'] = parmData['RESOLVE'] newRecord['RELATE'] = parmData['RELATE'] newRecord['REF_SCORE'] = int(parmData['REF_SCORE']) newRecord['RTYPE_ID'] = int(parmData['RTYPE_ID']) newRecord['QUAL_ERFRAG_CODE'] = parmData['FRAGMENT'] newRecord['DISQ_ERFRAG_CODE'] = storeNullableJsonString(parmData['DISQUALIFIER']) - newRecord['ERRULE_TIER'] = storeNullableJsonNumeric(parmData['TIER']) - + newRecord['ERRULE_TIER'] = parmData['TIER'] self.cfgData['G2_CONFIG']['CFG_ERRULE'].append(newRecord) self.configUpdated = True @@ -4184,7 +4345,6 @@ def do_addRule(self, arg): if self.doDebug: debug(newRecord) - # ===== system parameters ===== def do_listSystemParameters(self, arg): @@ -4195,7 +4355,6 @@ def do_listSystemParameters(self, arg): print(f'\n{{"relationshipsBreakMatches": "{i["BREAK_RES"]}"}}\n') break - def do_setSystemParameter(self, arg): '\n\tsetSystemParameter {"parameter": ""}\n' @@ -4203,7 +4362,7 @@ def do_setSystemParameter(self, arg): if not argCheck('templateAdd', arg, self.do_setSystemParameter.__doc__): return try: - parmData = json.loads(arg) #--don't want these upper + parmData = json.loads(arg) # --don't want these upper except (ValueError, KeyError) as e: argError(arg, e) return @@ -4213,7 +4372,7 @@ def do_setSystemParameter(self, arg): parameterValue = parmData[parameterCode] if parameterCode not in validParameters: - printWithNewLines( '%s is an invalid system parameter' % parameterCode, 'B') + printWithNewLines('%s is an invalid system parameter' % parameterCode, 'B') #--set all disclosed relationship types to break or not break matches elif parameterCode == 'relationshipsBreakMatches': @@ -4222,7 +4381,7 @@ def do_setSystemParameter(self, arg): elif parameterValue.upper() in ('NO', 'N'): breakRes = 0 else: - printWithNewLines( '%s is an invalid parameter for %s' % (parameterValue, parameterCode), 'B') + printWithNewLines('%s is an invalid parameter for %s' % (parameterValue, parameterCode), 'B') return for i in range(len(self.cfgData['G2_CONFIG']['CFG_RTYPE'])): @@ -4230,7 +4389,6 @@ def do_setSystemParameter(self, arg): self.cfgData['G2_CONFIG']['CFG_RTYPE'][i]['BREAK_RES'] = breakRes self.configUpdated = True - def do_touch(self, arg): '\n\tMarks configuration object as modified when no configuration changes have been applied yet.\n' @@ -4238,446 +4396,56 @@ def do_touch(self, arg): self.configUpdated = True print() +# ===== match levels ===== -# ===== database functions ===== - + def do_listMatchLevels(self, arg): + '\n\tlistMatchLevels [search_value]\n' - def do_updateDatabase(self, arg): - '\n\tInternal Senzing use!\n' - - if not self.g2Dbo: - printWithNewLines('ERROR: Database connectivity isn\'t available.', 'S') - printWithNewLines(f' Error from G2Database: {dbErr}', 'E') - return - - if self.configUpdated and not self.forceMode: - printWithNewLines('WARN: Configuration has been updated but not saved. Please save first to avoid inconsistencies!' ,'B') - return - - print(f'\nUpdating attributes...') - - if self.getRecordList('CFG_ATTR'): - cols = ['ATTR_ID', 'ATTR_CODE', 'ATTR_CLASS', 'FTYPE_CODE', 'FELEM_CODE', 'FELEM_REQ', 'DEFAULT_VALUE', 'ADVANCED', 'INTERNAL'] - insertSql = f'insert into CFG_ATTR ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_ATTR'), key = lambda k: k['ATTR_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - - try: - g2Dbo.sqlExec('delete from CFG_ATTR') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - cols = ['ATTR_ID', 'ATTR_CODE', 'ATTR_CLASS', 'FTYPE_CODE', 'FELEM_CODE', 'FELEM_REQ', 'DEFAULT_VALUE', 'ADVANCED'] - insertSql = f'insert into CFG_ATTR ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_ATTR'), key = lambda k: k['ATTR_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_ATTR') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_ATTR hasn\'t been updated: {err}', 'B') - - - print('Updating data sources...') - - if self.getRecordList('CFG_DSRC_INTEREST'): - cols = ['DSRC_ID', 'DSRC_CODE', 'DSRC_DESC', 'DSRC_RELY', 'RETENTION_LEVEL', 'CONVERSATIONAL'] - insertSql = f'insert into CFG_DSRC ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_DSRC'), key = lambda k: k['DSRC_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_DSRC') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_DSRC hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_DSRC_INTEREST'): - cols = ['DSRC_ID', 'MAX_DEGREE', 'INTEREST_FLAG'] - insertSql = f'insert into CFG_DSRC_INTEREST ({", ".join(map(str, cols))}) values (?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_DSRC_INTEREST'), key = lambda k: k['DSRC_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_DSRC_INTEREST') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_DSRC_INTEREST hasn\'t been updated: {err}', 'B') - - - print('Updating entity classes...') - - if self.getRecordList('CFG_ECLASS'): - cols = ['ECLASS_ID', 'ECLASS_CODE', 'ECLASS_DESC', 'RESOLVE'] - insertSql = f'insert into CFG_ECLASS ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_ECLASS'), key = lambda k: k['ECLASS_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_ECLASS') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_ECLASS hasn\'t been updated: {err}', 'B') - - - print('Updating entity types...') - - if self.getRecordList('CFG_ETYPE'): - cols = ['ETYPE_ID', 'ETYPE_CODE', 'ETYPE_DESC', 'ECLASS_ID'] - insertSql = f'insert into CFG_ETYPE ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_ETYPE'), key = lambda k: k['ETYPE_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_ETYPE') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_ETYPE hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_EBOM'): - cols = ['ETYPE_ID', 'EXEC_ORDER', 'FTYPE_ID', 'UTYPE_CODE'] - insertSql = f'insert into CFG_EBOM ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_EBOM'), key = lambda k: k['ETYPE_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_EBOM') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_EBOM hasn\'t been updated: {err}', 'B') - - - print('Updating features...') - - if self.getRecordList('CFG_FTYPE'): - cols = ['FTYPE_ID', 'FTYPE_CODE', 'FTYPE_DESC', 'FCLASS_ID', 'FTYPE_FREQ', 'FTYPE_STAB', 'FTYPE_EXCL', 'ANONYMIZE', 'DERIVED', 'USED_FOR_CAND', 'PERSIST_HISTORY', 'RTYPE_ID', 'VERSION'] - insertSql = f'insert into CFG_FTYPE ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_FTYPE'), key = lambda k: k['FTYPE_ID']): - if not 'FTYPE_DESC' in jsonRecord: - jsonRecord['FTYPE_DESC'] = jsonRecord['FTYPE_CODE'] - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_FTYPE') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_FTYPE hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_FBOM'): - cols = ['FTYPE_ID', 'FELEM_ID', 'EXEC_ORDER', 'DISPLAY_DELIM', 'DISPLAY_LEVEL', 'DERIVED'] - insertSql = f'insert into CFG_FBOM ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_FBOM'), key = lambda k: k['FTYPE_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_FBOM') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_FBOM hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_FELEM'): - cols = ['FELEM_ID', 'FELEM_CODE', 'TOKENIZE', 'DATA_TYPE'] - insertSql = f'insert into CFG_FELEM ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_FELEM'), key = lambda k: k['FELEM_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_FELEM') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_FELEM hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_SFCALL'): - cols = ['SFCALL_ID', 'SFUNC_ID', 'EXEC_ORDER', 'FTYPE_ID', 'FELEM_ID'] - insertSql = f'insert into CFG_SFCALL ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_SFCALL'), key = lambda k: k['SFCALL_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_SFCALL') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_SFCALL hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_SFUNC'): - cols = ['SFUNC_ID', 'SFUNC_CODE', 'SFUNC_DESC', 'FUNC_LIB', 'FUNC_VER', 'CONNECT_STR'] - insertSql = f'insert into CFG_SFUNC ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_SFUNC'), key = lambda k: k['SFUNC_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_SFUNC') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_SFUNC hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_EFCALL'): - cols = ['EFCALL_ID', 'EFUNC_ID', 'EXEC_ORDER', 'FTYPE_ID', 'FELEM_ID', 'EFEAT_FTYPE_ID', 'IS_VIRTUAL'] - insertSql = f'insert into CFG_EFCALL ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_EFCALL'), key = lambda k: k['EFCALL_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_EFCALL') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_EFCALL hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_EFUNC'): - cols = ['EFUNC_ID', 'EFUNC_CODE', 'EFUNC_DESC', 'FUNC_LIB', 'FUNC_VER', 'CONNECT_STR'] - insertSql = f'insert into CFG_EFUNC ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_EFUNC'), key = lambda k: k['EFUNC_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_EFUNC') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_EFUNC hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_EFBOM'): - cols = ['EFCALL_ID', 'EXEC_ORDER', 'FTYPE_ID', 'FELEM_ID', 'FELEM_REQ'] - insertSql = f'insert into CFG_EFBOM ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_EFBOM'), key = lambda k: k['EFCALL_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_EFBOM') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_EFBOM hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_CFCALL'): - cols = ['CFCALL_ID', 'CFUNC_ID', 'EXEC_ORDER', 'FTYPE_ID'] - insertSql = f'insert into CFG_CFCALL ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_CFCALL'), key = lambda k: k['CFCALL_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_CFCALL') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_CFCALL hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_CFBOM'): - cols = ['CFCALL_ID', 'EXEC_ORDER', 'FTYPE_ID', 'FELEM_ID'] - insertSql = f'insert into CFG_CFBOM ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_CFBOM'), key = lambda k: k['CFCALL_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_CFBOM') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_CFBOM hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_FBOVR'): - cols = ['FTYPE_ID', 'ECLASS_ID', 'UTYPE_CODE', 'FTYPE_FREQ', 'FTYPE_EXCL', 'FTYPE_STAB'] - insertSql = f'insert into CFG_FBOVR ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_FBOVR'), key = lambda k: k['FTYPE_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_FBOVR') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_FBOVR hasn\'t been updated: {err}', 'B') - - - print('Updating feature classes...') - - if self.getRecordList('CFG_FCLASS'): - cols = ['FCLASS_ID', 'FCLASS_CODE', 'FCLASS_DESC'] - insertSql = f'insert into CFG_FCLASS ({", ".join(map(str, cols))}) values (?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_FCLASS'), key = lambda k: k['FCLASS_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_FCLASS') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_FCLASS hasn\'t been updated: {err}', 'B') - - - print('Updating relationships...') - - if self.getRecordList('CFG_RTYPE'): - cols = ['RTYPE_ID', 'RTYPE_CODE', 'RCLASS_ID', 'REL_STRENGTH', 'BREAK_RES'] - insertSql = f'insert into CFG_RTYPE ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_RTYPE'), key = lambda k: k['RTYPE_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_RTYPE') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_RTYPE hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_RCLASS'): - cols = ['RCLASS_ID', 'RCLASS_CODE', 'RCLASS_DESC', 'IS_DISCLOSED'] - insertSql = f'insert into CFG_RCLASS ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_RCLASS'), key = lambda k: k['RCLASS_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_RCLASS') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_RCLASS hasn\'t been updated: {err}', 'B') - - - print('Updating resolution rules...') - - if self.getRecordList('CFG_ERFRAG'): - cols = ['ERFRAG_ID', 'ERFRAG_CODE', 'ERFRAG_DESC', 'ERFRAG_SOURCE', 'ERFRAG_DEPENDS'] - insertSql = f'insert into CFG_ERFRAG ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_ERFRAG'), key = lambda k: k['ERFRAG_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_ERFRAG') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_ERFRAG hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_ERRULE'): - cols = ['ERRULE_ID', 'ERRULE_CODE', 'ERRULE_DESC', 'RESOLVE', 'RELATE', 'REF_SCORE', 'RTYPE_ID', 'QUAL_ERFRAG_CODE', 'DISQ_ERFRAG_CODE', 'ERRULE_TIER'] - insertSql = f'insert into CFG_ERRULE ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_ERRULE'), key = lambda k: k['ERRULE_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_ERRULE') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_ERRULE hasn\'t been updated: {err}', 'B') - - - print('Updating comps...') + print() + for rtypeRecord in sorted(self.getRecordList('CFG_RTYPE'), key=lambda k: k['RTYPE_ID']): + if arg and arg.lower() not in str(rtypeRecord).lower(): + continue + print(f'{{"level": {rtypeRecord["RTYPE_ID"]}, "code": "{rtypeRecord["RTYPE_CODE"]}", "class": "{self.getRecord("CFG_RCLASS", "RCLASS_ID", rtypeRecord["RCLASS_ID"])["RCLASS_DESC"]}"}}') + print() - if self.getRecordList('CFG_CFUNC'): - cols = ['ANON_SUPPORT', 'CFUNC_CODE', 'CFUNC_DESC', 'CFUNC_ID', 'CONNECT_STR', 'FUNC_LIB', 'FUNC_VER'] - insertSql = f'insert into CFG_CFUNC ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_CFUNC'), key = lambda k: k['CFUNC_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_CFUNC') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_CFUNC hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_CFRTN'): - cols = ['CFRTN_ID', 'CFUNC_ID', 'CFUNC_RTNVAL', 'EXEC_ORDER', 'SAME_SCORE', 'CLOSE_SCORE', 'LIKELY_SCORE', 'PLAUSIBLE_SCORE', 'UN_LIKELY_SCORE'] - insertSql = f'insert into CFG_CFRTN ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_CFRTN'), key = lambda k: k['CFRTN_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_CFRTN') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_CFRTN hasn\'t been updated: {err}', 'B') +# ===== Class Utils ===== - print('Updating distict...') + def printResponse(self, response): - if self.getRecordList('CFG_DFCALL'): - cols = ['DFCALL_ID', 'FTYPE_ID', 'DFUNC_ID', 'EXEC_ORDER'] - insertSql = f'insert into CFG_DFCALL ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_DFCALL'), key = lambda k: k['DFCALL_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_DFCALL') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_DFCALL hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_DFUNC'): - cols = ['DFUNC_ID', 'DFUNC_CODE', 'DFUNC_DESC', 'FUNC_LIB', 'FUNC_VER', 'CONNECT_STR','ANON_SUPPORT'] - insertSql = f'insert into CFG_DFUNC ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_DFUNC'), key = lambda k: k['DFUNC_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_DFUNC') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_DFUNC hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_DFBOM'): - cols = ['DFCALL_ID', 'FTYPE_ID', 'FELEM_ID', 'EXEC_ORDER'] - insertSql = f'insert into CFG_DFBOM ({", ".join(map(str, cols))}) values (?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_DFBOM'), key = lambda k: k['DFCALL_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_DFBOM') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_DFBOM hasn\'t been updated: {err}', 'B') + if response: + self.jsonOutput(response.decode().rstrip()) + else: + printWithNewLines('Empty response!', 'B') + def printJsonResponse(self, response): + ''' ''' - print('Updating generics...') + if response: + response = response.decode() if isinstance(response, bytearray) else response - if self.getRecordList('CFG_GENERIC_THRESHOLD'): - cols = ['GPLAN_ID', 'BEHAVIOR', 'FTYPE_ID', 'CANDIDATE_CAP', 'SCORING_CAP', 'SEND_TO_REDO'] - insertSql = f'insert into CFG_GENERIC_THRESHOLD ({", ".join(map(str, cols))}) values (?, ?, ?, ?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_GENERIC_THRESHOLD'), key = lambda k: k['GPLAN_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_GENERIC_THRESHOLD') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_GENERIC_THRESHOLD hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_GPLAN'): - cols = ['GPLAN_ID', 'GPLAN_CODE', 'GPLAN_DESC'] - insertSql = f'insert into CFG_GPLAN ({", ".join(map(str, cols))}) values (?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_GPLAN'), key = lambda k: k['GPLAN_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_GPLAN') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_GPLAN hasn\'t been updated: {err}', 'B') + if self.pygmentsInstalled: + if self.prettyPrint: + print('\n' + highlight(json.dumps(json.loads(response), indent=2), lexers.JsonLexer(), formatters.TerminalFormatter())) + return + else: + if self.prettyPrint: + print(f'\n {json.dumps(json.loads(response), indent=2)} \n') + return + printWithNewLines(f'{response}', 'B') + return - print('Updating lens...') + printWithNewLines('Empty response!', 'B') - if self.getRecordList('CFG_LENS'): - cols = ['LENS_ID', 'LENS_CODE', 'LENS_DESC'] - insertSql = f'insert into CFG_LENS ({", ".join(map(str, cols))}) values (?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_LENS'), key = lambda k: k['LENS_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_LENS') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_LENS hasn\'t been updated: {err}', 'B') - - if self.getRecordList('CFG_DSRC_INTEREST'): - cols = ['LENS_ID', 'ERRULE_ID', 'EXEC_ORDER'] - insertSql = f'insert into CFG_LENSRL ({", ".join(map(str, cols))}) values (?, ?, ?)' - insertRecords = [] - for jsonRecord in sorted(self.getRecordList('CFG_LENSRL'), key = lambda k: k['LENS_ID']): - [ insertRecords.append([jsonRecord[k] for k in cols]) ] - try: - g2Dbo.sqlExec('delete from CFG_LENSRL') - g2Dbo.execMany(insertSql, insertRecords) - except G2Exception.G2DBException as err: - printWithNewLines(f'ERROR: CFG_LENSRL hasn\'t been updated: {err}', 'B') - print() +# ===== Utility functions ===== +def parse(argumentString): + 'Parses an argument list into a logical set of argument strings' + return shlex.split(argumentString) -# ===== Utility functions ===== def getFeatureBehavior(feature): @@ -4724,21 +4492,31 @@ def argError(errorArg, error): def printWithNewLines(ln, pos=''): pos = pos.upper() - if pos == 'S' or pos == 'START' : + if pos == 'S' or pos == 'START': print(f'\n{ln}') - elif pos == 'E' or pos == 'END' : + elif pos == 'E' or pos == 'END': print(f'{ln}\n') - elif pos == 'B' or pos == 'BOTH' : + elif pos == 'B' or pos == 'BOTH': print(f'\n{ln}\n') else: print(f'{ln}') +def printResponse(response): + + if response: + response = response.decode() if isinstance(response, bytearray) else response + print(f'\n{response}\n') + return + + printWithNewLines('Empty response!', 'B') + + def dictKeysUpper(dictionary): if isinstance(dictionary, list): return [v.upper() for v in dictionary] elif isinstance(dictionary, dict): - return {k.upper():v for k,v in dictionary.items()} + return {k.upper(): v for k, v in dictionary.items()} else: return dictionary @@ -4749,18 +4527,21 @@ def showNullableJsonString(val): else: return '"%s"' % val + def showNullableJsonNumeric(val): if not val: return 'null' else: return '%s' % val + def storeNullableJsonString(val): if not val or val == 'null': return None else: return val + def storeNullableJsonNumeric(val): if not val or val == 'null': return None @@ -4779,43 +4560,32 @@ def debug(data, loc=''): '''), 'E') - -# ===== The main function ===== if __name__ == '__main__': argParser = argparse.ArgumentParser() argParser.add_argument("fileToProcess", default=None, nargs='?') - argParser.add_argument('-c', '--ini-file-name', dest='ini_file_name', default=None, help='name of the g2.ini file') + argParser.add_argument('-c', '--ini-file-name', dest='ini_file_name', default=None, help='name of a G2Module.ini file to use') argParser.add_argument('-f', '--force', dest='forceMode', default=False, action='store_true', help='when reading from a file, execute each command without prompts') argParser.add_argument('-H', '--histDisable', dest='histDisable', action='store_true', default=False, help='disable history file usage') args = argParser.parse_args() - iniFileName = G2Paths.get_G2Module_ini_path() if not args.ini_file_name else args.ini_file_name - - if not os.path.exists(iniFileName): - raise FileNotFoundError(f'INI file {iniFileName} not found') + # If ini file isn't specified try and locate it with G2Paths + ini_file_name = pathlib.Path(G2Paths.get_G2Module_ini_path()) if not args.ini_file_name else pathlib.Path(args.ini_file_name).resolve() + G2Paths.check_file_exists_and_readable(ini_file_name) + # Warn if using out dated parms g2health = G2Health() - g2health.checkIniParams(iniFileName) + g2health.checkIniParams(ini_file_name) # Older Senzing using G2CONFIGFILE, e.g, G2CONFIGFILE=/opt/senzing/g2/python/g2config.json iniParamCreator = G2IniParams() - g2ConfigFile = iniParamCreator.getINIParam(iniFileName,'SQL','G2CONFIGFILE') + g2ConfigFile = iniParamCreator.getINIParam(ini_file_name, 'SQL', 'G2CONFIGFILE') - # Is there database support? - g2dbUri = iniParamCreator.getINIParam(iniFileName,'SQL','CONNECTION') + # Get the INI paramaters to use + g2module_params = iniParamCreator.getJsonINIParams(ini_file_name) - if not g2dbUri: - printWithNewLines(f'CONNECTION parameter not found in [SQL] section of {iniFileName} file') - sys.exit(1) - else: - try: - g2Dbo = G2Database(g2dbUri) - except Exception as err: - g2Dbo = False - dbErr = err + cmd_obj = G2CmdShell(g2ConfigFile, args.histDisable, args.forceMode, args.fileToProcess, ini_file_name) - cmd_obj = G2CmdShell(g2ConfigFile, args.histDisable, args.forceMode, args.fileToProcess, iniFileName, g2Dbo) if args.fileToProcess: cmd_obj.fileloop() else: diff --git a/g2/python/G2Explorer.py b/g2/python/G2Explorer.py index 6670d5f..dccd474 100755 --- a/g2/python/G2Explorer.py +++ b/g2/python/G2Explorer.py @@ -7,12 +7,13 @@ import os import platform import re -import subprocess import sys import textwrap import traceback from collections import OrderedDict import configparser +import subprocess +import tempfile try: import readline import atexit @@ -97,12 +98,13 @@ def colorize(string, colorList = None): class ColoredTable(prettytable.PrettyTable): def __init__(self, field_names=None, **kwargs): - new_options = ['title_color', 'header_color'] + new_options = ['title_color', 'header_color', 'title_justify'] super(ColoredTable, self).__init__(field_names, **kwargs) self._title_color = kwargs['title_color'] or None self._header_color = kwargs['header_color'] or None + self._title_justify = kwargs['title_justify'] or 'c' self._options.extend(new_options) @@ -125,10 +127,11 @@ def _stringify_title(self, title, options): endpoint = options["vertical_char"] if options["vrules"] in (self.ALL, self.FRAME) else " " bits.append(endpoint) title = " " * lpad + title + " " * rpad + if options['title_color']: - bits.append(colorize(self._justify(title, len(self._hrule) - 2, "c"), options['title_color'])) + bits.append(colorize(self._justify(title, len(self._hrule) - 2, options['title_justify']), options['title_color'])) else: - bits.append(self._justify(title, len(self._hrule) - 2, "c")) + bits.append(self._justify(title, len(self._hrule) - 2, options['title_justify'])) bits.append(endpoint) lines.append("".join(bits)) @@ -326,10 +329,6 @@ def __init__(self): self.auditFile = None self.auditData = {} - #--set the last table name - self.lastTableName = os.path.join(os.path.expanduser("~"), 'lastTable.txt') - self.lastTableData = None - #--history self.readlineAvail = True if 'readline' in sys.modules else False self.histDisable = hist_disable @@ -632,7 +631,7 @@ def xx_perfStats (self,arg): # ----------------------------- def do_quickLook (self,arg): - '\nDisplays whats in the current database without a snapshot' + '\nDisplays current data source stats without a snapshot' g2_diagnostic_module = G2Diagnostic() g2_diagnostic_module.initV2('pyG2Diagnostic', iniParams, False) @@ -1941,7 +1940,7 @@ def do_search(self,arg): matchList[i][0] = str(i+1) matchList[i][1] = colorize(matchList[i][1], self.colors['entityid']) matchList[i][2] = matchList[i][2] - self.renderTable(tblTitle, tblColumns, matchList, pageRecords=10) + self.renderTable(tblTitle, tblColumns, matchList) print('') @@ -1980,6 +1979,7 @@ def do_get(self,arg): arg = str(self.lastSearchResult[int(lastToken)-1]) getFlags = 0 + if apiVersion['VERSION'][0:1] > '1': #getFlags = g2Engine.G2_ENTITY_DEFAULT_FLAGS getFlags = getFlags | g2Engine.G2_ENTITY_INCLUDE_ENTITY_NAME @@ -2023,18 +2023,89 @@ def do_get(self,arg): if len(response) == 0: printWithNewLines('0 records found %s' % response, 'B') return -1 if calledDirect else 0 + + resolvedJson = json.loads(str(response)) + relatedEntityCount = len(resolvedJson['RELATED_ENTITIES']) if 'RELATED_ENTITIES' in resolvedJson else 0 + entityID = str(resolvedJson['RESOLVED_ENTITY']['ENTITY_ID']) + entityName = resolvedJson['RESOLVED_ENTITY']['ENTITY_NAME'] + + reportType = 'Detail' if showDetail else 'Summary' + tblTitle = f'Entity {reportType} for: {entityID} - {entityName}' + tblColumns = [] + tblColumns.append({'name': 'Record ID', 'width': 50, 'align': 'left'}) + tblColumns.append({'name': 'Entity Data', 'width': 100, 'align': 'left'}) + tblColumns.append({'name': 'Additional Data', 'width': 100, 'align': 'left'}) + + #--summarize by data source + if reportType == 'Summary': + dataSources = {} + recordList = [] + for record in resolvedJson['RESOLVED_ENTITY']['RECORDS']: + if record['DATA_SOURCE'] not in dataSources: + dataSources[record['DATA_SOURCE']] = [] + dataSources[record['DATA_SOURCE']].append(record) + + #--summarize by data source + for dataSource in sorted(dataSources): + recordData, entityData, otherData = self.formatRecords(dataSources[dataSource], reportType) + row = [recordData, entityData, otherData] + recordList.append(row) + + #--display each record else: - resolvedJson = json.loads(str(response)) + recordList = [] + for record in sorted(resolvedJson['RESOLVED_ENTITY']['RECORDS'], key = lambda k: (k['DATA_SOURCE'], k['RECORD_ID'])): + recordData, entityData, otherData = self.formatRecords(record, 'entityDetail') + row = [recordData, entityData, otherData] + recordList.append(row) + + #--display if no relationships + if relatedEntityCount == 0: + self.renderTable(tblTitle, tblColumns, recordList, titleColor=self.colors['entityTitle']) + return 0 - if showDetail: - self.showEntityDetail(resolvedJson) - else: - self.showEntitySummary(resolvedJson) + #--otherwise begin the report and add the relationships + self.renderTable(tblTitle, tblColumns, recordList, titleColor=self.colors['entityTitle'], displayFlag='begin') + + relationships = [] + for relatedEntity in resolvedJson['RELATED_ENTITIES']: + relationship = {} + relationship['MATCH_LEVEL'] = relatedEntity['MATCH_LEVEL'] + relationship['MATCH_SCORE'] = relatedEntity['MATCH_SCORE'] + relationship['MATCH_KEY'] = relatedEntity['MATCH_KEY'] + relationship['ERRULE_CODE'] = relatedEntity['ERRULE_CODE'] + relationship['ENTITY_ID'] = relatedEntity['ENTITY_ID'] + relationship['ENTITY_NAME'] = relatedEntity['ENTITY_NAME'] + relationship['DATA_SOURCES'] = [] + for dataSource in relatedEntity['RECORD_SUMMARY']: + relationship['DATA_SOURCES'].append('%s (%s)' %(colorize(dataSource['DATA_SOURCE'], self.colors['datasource']), dataSource['RECORD_COUNT'])) + relationships.append(relationship) + tblTitle = f'{relatedEntityCount} related entities' + tblColumns = [] + tblColumns.append({'name': 'Entity ID', 'width': 15, 'align': 'left'}) + tblColumns.append({'name': 'Entity Name', 'width': 75, 'align': 'left'}) + tblColumns.append({'name': 'Data Sources', 'width': 75, 'align': 'left'}) + tblColumns.append({'name': 'Match Level', 'width': 25, 'align': 'left'}) + tblColumns.append({'name': 'Match Key', 'width': 50, 'align': 'left'}) + relatedRecordList = [] + for relationship in sorted(relationships, key = lambda k: k['MATCH_LEVEL']): + row = [] + row.append(colorize(str(relationship['ENTITY_ID']), self.colors['entityid'])) + row.append(relationship['ENTITY_NAME']) + row.append('\n'.join(sorted(relationship['DATA_SOURCES']))) + row.append(self.relatedMatchLevels[relationship['MATCH_LEVEL']]) + matchData = {} + matchData['matchKey'] = relationship['MATCH_KEY'] + matchData['ruleCode'] = self.getRuleDesc(relationship['ERRULE_CODE']) + row.append(formatMatchData(matchData, self.colors)) + relatedRecordList.append(row) + + self.renderTable(tblTitle, tblColumns, relatedRecordList, titleColor=self.colors['entityTitle'], titleJustify='l', displayFlag='end') return 0 # ----------------------------- - def formatRecords(self, recordList, reportName): + def formatRecords(self, recordList, reportType): dataSource = 'unknown' recordIdList = [] primaryNameList = [] @@ -2050,7 +2121,7 @@ def formatRecords(self, recordList, reportName): dataSource = colorize(record['DATA_SOURCE'], self.colors['datasource']) recordIdData = record['RECORD_ID'] - if reportName == 'entityDetail': + if reportType == 'Detail': if record['MATCH_KEY']: matchData = {} matchData['matchKey'] = record['MATCH_KEY'] @@ -2074,17 +2145,17 @@ def formatRecords(self, recordList, reportName): for item in record['IDENTIFIER_DATA']: identifierList.append(colorizeAttribute(item, self.colors['highlight1'])) for item in sorted(record['OTHER_DATA']): - if not self.isInternalAttribute(item) or reportName == 'entityDetail': + if not self.isInternalAttribute(item) or reportType == 'Detail': otherList.append(colorizeAttribute(item, self.colors['highlight1'])) recordDataList = [dataSource] + sorted(recordIdList) entityDataList = list(set(primaryNameList)) + list(set(otherNameList)) + sorted(set(attributeList)) + sorted(set(identifierList)) + list(set(addressList)) + list(set(phoneList)) otherDataList = sorted(set(otherList)) - if reportName == 'entitySummary': - columnHeightLimit = 50 - else: + if reportType == 'Detail': columnHeightLimit = 1000 + else: + columnHeightLimit = 50 recordData = '\n'.join(recordDataList[:columnHeightLimit]) if len(recordDataList) > columnHeightLimit: @@ -2100,103 +2171,6 @@ def formatRecords(self, recordList, reportName): return recordData, entityData, otherData - # ----------------------------- - def showEntitySummary(self, resolvedJson): - - entityID = str(resolvedJson['RESOLVED_ENTITY']['ENTITY_ID']) - tblTitle = 'Entity Summary for: %s - %s' % (entityID, resolvedJson['RESOLVED_ENTITY']['ENTITY_NAME']) - tblColumns = [] - tblColumns.append({'name': 'Record ID', 'width': 50, 'align': 'left'}) - tblColumns.append({'name': 'Entity Data', 'width': 100, 'align': 'left'}) - tblColumns.append({'name': 'Additional Data', 'width': 100, 'align': 'left'}) - - #--group by data source - dataSources = {} - recordList = [] - for record in resolvedJson['RESOLVED_ENTITY']['RECORDS']: - if record['DATA_SOURCE'] not in dataSources: - dataSources[record['DATA_SOURCE']] = [] - dataSources[record['DATA_SOURCE']].append(record) - - #--summarize by data source - for dataSource in sorted(dataSources): - recordData, entityData, otherData = self.formatRecords(dataSources[dataSource], 'entitySummary') - row = [recordData, entityData, otherData] - recordList.append(row) - self.renderTable(tblTitle, tblColumns, recordList, titleColor=self.colors['entityTitle']) - - #--show relationships if there are any and not reviewing a list - if 'RELATED_ENTITIES' in resolvedJson and len(resolvedJson['RELATED_ENTITIES']) > 0 and not self.currentReviewList: - self.showRelatedEntities(resolvedJson) - - - # ----------------------------- - def showEntityDetail(self, resolvedJson): - - tblTitle = 'Entity Detail for: %s - %s' % (resolvedJson['RESOLVED_ENTITY']['ENTITY_ID'], resolvedJson['RESOLVED_ENTITY']['ENTITY_NAME']) - tblColumns = [] - tblColumns.append({'name': 'Record ID', 'width': 50, 'align': 'left'}) - tblColumns.append({'name': 'Entity Data', 'width': 100, 'align': 'left'}) - tblColumns.append({'name': 'Additional Data', 'width': 100, 'align': 'left'}) - - recordList = [] - for record in sorted(resolvedJson['RESOLVED_ENTITY']['RECORDS'], key = lambda k: (k['DATA_SOURCE'], k['RECORD_ID'])): - recordData, entityData, otherData = self.formatRecords(record, 'entityDetail') - row = [recordData, entityData, otherData] - recordList.append(row) - self.renderTable(tblTitle, tblColumns, recordList, pageRecords=5, titleColor=self.colors['entityTitle']) - - #--not trying to analyze entities here - if 'RELATED_ENTITIES' in resolvedJson and len(resolvedJson['RELATED_ENTITIES']) > 0: - self.showRelatedEntities(resolvedJson) - - # ----------------------------- - def showRelatedEntities(self, resolvedJson): - - #--determine what, if any relationships exist - relationships = [] - for relatedEntity in resolvedJson['RELATED_ENTITIES']: - relationship = {} - relationship['MATCH_LEVEL'] = relatedEntity['MATCH_LEVEL'] - relationship['MATCH_SCORE'] = relatedEntity['MATCH_SCORE'] - relationship['MATCH_KEY'] = relatedEntity['MATCH_KEY'] - relationship['ERRULE_CODE'] = relatedEntity['ERRULE_CODE'] - relationship['ENTITY_ID'] = relatedEntity['ENTITY_ID'] - relationship['ENTITY_NAME'] = relatedEntity['ENTITY_NAME'] - relationship['DATA_SOURCES'] = [] - for dataSource in relatedEntity['RECORD_SUMMARY']: - relationship['DATA_SOURCES'].append('%s (%s)' %(colorize(dataSource['DATA_SOURCE'], self.colors['datasource']), dataSource['RECORD_COUNT'])) - relationships.append(relationship) - - reply = input('%s relationships found, press D to display %s or enter to skip ... ' % (len(relationships), ('it' if len(relationships) ==1 else 'them'))) - if reply: - removeFromHistory() - print('') - - if reply.upper().startswith('D'): - - tblTitle = 'Entities related to: %s - %s' % (resolvedJson['RESOLVED_ENTITY']['ENTITY_ID'], resolvedJson['RESOLVED_ENTITY']['ENTITY_NAME']) - tblColumns = [] - tblColumns.append({'name': 'Entity ID', 'width': 15, 'align': 'left'}) - tblColumns.append({'name': 'Entity Name', 'width': 75, 'align': 'left'}) - tblColumns.append({'name': 'Data Sources', 'width': 75, 'align': 'left'}) - tblColumns.append({'name': 'Match Level', 'width': 25, 'align': 'left'}) - tblColumns.append({'name': 'Match Key', 'width': 50, 'align': 'left'}) - relatedRecordList = [] - for relationship in sorted(relationships, key = lambda k: k['MATCH_LEVEL']): - row = [] - row.append(colorize(str(relationship['ENTITY_ID']), self.colors['entityid'])) - row.append(relationship['ENTITY_NAME']) - row.append('\n'.join(sorted(relationship['DATA_SOURCES']))) - row.append(self.relatedMatchLevels[relationship['MATCH_LEVEL']]) - matchData = {} - matchData['matchKey'] = relationship['MATCH_KEY'] - matchData['ruleCode'] = self.getRuleDesc(relationship['ERRULE_CODE']) - row.append(formatMatchData(matchData, self.colors)) - relatedRecordList.append(row) - - self.renderTable(tblTitle, tblColumns, relatedRecordList, titleColor=self.colors['entityTitle']) - # ----------------------------- def getAmbiguousEntitySet(self, entityId): #--get other ambiguous relationships if this is the ambiguous entity @@ -3261,9 +3235,13 @@ def do_score(self, arg): # ----------------------------- def renderTable(self, tblTitle, tblColumns, tblRows, **kwargs): + #--display flags (start/append/done) allow for multiple tables to be displayed together and scrolled as one + #--such as an entity and its relationships + #--possible kwargs - pageRecords = kwargs['pageRecords'] if 'pageRecords' in kwargs else 0 + displayFlag = kwargs['displayFlag'] if 'displayFlag' in kwargs else None titleColor = kwargs['titleColor'] if 'titleColor' in kwargs else self.colors['tableTitle'] + titleJustify = kwargs['titleJustify'] if 'titleJustify' in kwargs else 'l' #--left headerColor = kwargs['headerColor'] if 'headerColor' in kwargs else self.colors['columnHeader'] #--setup the table @@ -3273,83 +3251,67 @@ def renderTable(self, tblTitle, tblColumns, tblRows, **kwargs): tableWidth += tblColumns[i]['width'] tblColumns[i]['name'] = str(tblColumns[i]['name']) columnHeaderList.append(tblColumns[i]['name']) - tableObject = ColoredTable(title_color=titleColor, header_color=headerColor) + tableObject = ColoredTable(title_color=titleColor, header_color=headerColor, title_justify=titleJustify) tableObject.hrules = prettytable.ALL tableObject.title = tblTitle tableObject.field_names = columnHeaderList thisTable = tableObject.copy() totalRowCnt = 0 - tableRowCnt = 0 for row in tblRows: totalRowCnt += 1 - tableRowCnt += 1 row[0] = '\n'.join([i for i in str(row[0]).split('\n')]) if self.usePrettyTable: thisTable.add_row(row) else: thisTable.append_row(row) - if pageRecords !=0 and tableRowCnt == pageRecords: - - #--write to last table so can be viewed with less if necessary - try: - with open(self.lastTableName,'w') as file: - file.write(thisTable.get_string()) - except: pass - self.lastTableData = thisTable.get_string() - - #--format with data in the table before printing - for columnData in tblColumns: - thisTable.max_width[str(columnData['name'])] = columnData['width'] - thisTable.align[str(columnData['name'])] = columnData['align'][0:1].lower() - print(thisTable) - thisTable = tableObject.copy() - tableRowCnt = 0 - - #--ask if they wanna continue if more to display - if len(tblRows) - totalRowCnt > 0: - print('') - reply = input('%s more records to display, press enter to continue or Q to quit ... ' % (len(tblRows) - totalRowCnt)) - print('') - if reply: - removeFromHistory() - if reply and reply.upper().startswith('Q'): - break - - #--print any remaining rows - if tableRowCnt > 0: + #--format with data in the table + for columnData in tblColumns: + thisTable.max_width[str(columnData['name'])] = columnData['width'] + thisTable.align[str(columnData['name'])] = columnData['align'][0:1].lower() + + #--write to a file so can be viewed with less + #--also write to the lastTableData variable in case canot write to file + fmtTableString = thisTable.get_string() + '\n' + writeMode = 'w' + if displayFlag in ('append', 'end'): + fmtTableString = '\n' + fmtTableString + writeMode = 'a' + + if writeMode == 'w': + self.currentRenderString = fmtTableString + else: + self.currentRenderString = self.currentRenderString + fmtTableString - #--format with data in the table + # display if a single table or done acculating tables to display + if not displayFlag or displayFlag == 'end': print('') if self.currentReviewList: print(colorize(self.currentReviewList, 'bold')) - for columnData in tblColumns: - thisTable.max_width[str(columnData['name'])] = columnData['width'] - thisTable.align[str(columnData['name'])] = columnData['align'][0:1].lower() - print(thisTable.get_string()) - - #--write to last table so can be viewed with less if necessary - try: - with open(self.lastTableName,'w') as file: - file.write(thisTable.get_string()) - except: pass - self.lastTableData = thisTable.get_string() - - if len(tblRows) - totalRowCnt == 0 and pageRecords != 0: #--signify done if paging and all rows displayed - print('') - print('%s rows returned, complete!' % totalRowCnt) - print('') + self.do_scroll('auto') return # ----------------------------- def do_scroll(self,arg): '\nLoads the last table rendered into the linux less viewer where you can use the arrow keys to scroll ' \ '\n up and down, left and right, until you type Q to quit.\n' - if os.path.exists(self.lastTableName): - os.system('less -SR %s' % self.lastTableName) + + #--note: the F allows less to auto quit if output fits on screen + #-- if they purposely went into scroll mode, we should not auto-quit! + if arg == 'auto': + lessOptions = 'FMXSR' else: - os.system('echo "%s" | less -SR ' % self.lastTableData) + lessOptions = 'MXSR' + + #--try pipe to less on small enough files (pipe buffer usually 1mb and fills up on large entity displays) + less = subprocess.Popen(["less", "-FMXSR"], stdin=subprocess.PIPE) + try: + less.stdin.write(self.currentRenderString.encode('utf-8')) + except IOError: + pass + less.stdin.close() + less.wait() # ----------------------------- def do_export(self,arg): @@ -3734,9 +3696,7 @@ def fuzzyCompare(ftypeCode, cfuncCode, str1, str2): g2ConfigMgr.destroy() #--cmdloop() - subprocess.Popen(["echo", "-ne", "\e[?7l"]) #--text wrapping off G2CmdShell().cmdloop() - subprocess.Popen(["echo", "-ne", "\e[?7h"]) #--text wrapping on print('') #--cleanups diff --git a/g2/python/G2Loader.py b/g2/python/G2Loader.py index b236d72..3867fed 100644 --- a/g2/python/G2Loader.py +++ b/g2/python/G2Loader.py @@ -61,7 +61,6 @@ def govern_cleanup(self, *args, **kwargs): return - #--------------------------------------------------------------------- # G2Loader #--------------------------------------------------------------------- @@ -232,7 +231,7 @@ def redoFeed(q, debugTrace, redoMode, redoModeInterval): return -def check_resources_and_startup(thread_count, doPurge, doLicense=True): +def check_resources_and_startup(returnQueue, thread_count, doPurge, doLicense=True): ''' Check system resources, calculate a safe number of threads when argument not specified on command line ''' try: @@ -241,14 +240,16 @@ def check_resources_and_startup(thread_count, doPurge, doLicense=True): except G2ModuleException as ex: print('\nERROR: Could not start G2Diagnostic for check_resources_and_startup()') print(f' {ex}') - sys.exit(1) + returnQueue.put(-1) + return try: g2_engine = init_engine('pyG2StartSetup', g2module_params, args.debugTrace, prime_engine=False) except G2ModuleException as ex: print('ERROR: Could not start the G2 engine for check_resources_and_startup()') print(f' {ex}') - sys.exit(1) + returnQueue.put(-1) + return try: g2_configmgr = G2ConfigMgr() @@ -256,7 +257,8 @@ def check_resources_and_startup(thread_count, doPurge, doLicense=True): except G2ModuleException as ex: print('ERROR: Could not start G2ConfigMgr for check_resources_and_startup()') print(f' {ex}') - sys.exit(1) + returnQueue.put(-1) + return try: g2_product = G2Product() @@ -264,7 +266,8 @@ def check_resources_and_startup(thread_count, doPurge, doLicense=True): except G2ModuleException as ex: print('ERROR: Could not start G2Product for check_resources_and_startup()') print(f' {ex}\n') - sys.exit(1) + returnQueue.put(-1) + return licInfo = json.loads(g2_product.license()) verInfo = json.loads(g2_product.version()) @@ -277,7 +280,8 @@ def check_resources_and_startup(thread_count, doPurge, doLicense=True): except G2Exception.G2Exception as ex: print('ERROR: Could not get config list in check_resources_and_startup()') print(f' {ex}') - sys.exit(1) + returnQueue.put(-1) + return # Get the active config ID try: @@ -287,7 +291,8 @@ def check_resources_and_startup(thread_count, doPurge, doLicense=True): except G2Exception.G2Exception as ex: print('ERROR: Could not get the active config in check_resources_and_startup()') print(f' {ex}') - sys.exit(1) + returnQueue.put(-1) + return # Get details for the currently active ID active_cfg_details = [details for details in config_list['CONFIGS'] if details['CONFIG_ID'] == active_cfg_id] @@ -465,20 +470,21 @@ def check_resources_and_startup(thread_count, doPurge, doLicense=True): print('\nPurging Senzing database...') g2_engine.purgeRepository(False) - # Clean up - g2_engine.destroy() - del g2_engine + # Clean up (in reverse order of initialization) + g2_product.destroy() + del g2_product g2_configmgr.destroy() del g2_configmgr - g2_product.destroy() - del g2_product + g2_engine.destroy() + del g2_engine diag.destroy() del diag - return thread_count + # Return values are put in a queue + returnQueue.put(thread_count) def perform_load(): @@ -490,22 +496,32 @@ def perform_load(): # Prepare the G2 configuration g2ConfigJson = bytearray() - if not getInitialG2Config(g2module_params, g2ConfigJson): + tempQueue = Queue() + getInitialG2ConfigProcess = Process(target=getInitialG2Config_processWrapper, args=(tempQueue,g2module_params, g2ConfigJson)) + getInitialG2ConfigProcess.start() + g2ConfigJson = tempQueue.get(block=True) + resultOfGetInitialG2Config = tempQueue.get(block=True) + getInitialG2ConfigProcess.join() + + if not resultOfGetInitialG2Config: return 1 g2ConfigTables = G2ConfigTables(g2ConfigJson) - g2Project = G2Project(g2ConfigTables, args.projectFileName, args.projectFileSpec, args.tmpPath) + g2Project = G2Project(g2ConfigTables, dsrcAction, args.projectFileName, args.projectFileSpec, args.tmpPath) if not g2Project.success: return 1 # Enhance the G2 configuration, by adding data sources and entity types - if not enhanceG2Config(g2Project, g2module_params, g2ConfigJson, args.configuredDatasourcesOnly): - return 1 - - # Purge log files created by g2 from prior runs - # for filename in glob('pyG2*'): - # os.remove(filename) + if not args.testMode: + tempQueue = Queue() + enhanceG2ConfigProcess = Process(target=enhanceG2Config_processWrapper, args=(tempQueue, g2Project, g2module_params, g2ConfigJson, args.configuredDatasourcesOnly)) + enhanceG2ConfigProcess.start() + g2ConfigJson = tempQueue.get(block=True) + resultOfEnhanceG2Config = tempQueue.get(block=True) + enhanceG2ConfigProcess.join() + if not resultOfEnhanceG2Config: + return 1 # Start loading for sourceDict in g2Project.sourceList: @@ -517,25 +533,15 @@ def perform_load(): cntRows = cntBadParse = cntBadUmf = cntGoodUmf = 0 g2Project.clearStatPack() - if not args.testMode: + if args.testMode: + print(f'\nTesting {filePath}, CTRL-C to end test at any time...\n') + else: if dsrcAction == 'D': print(f'\n{"-"*30} Deleting {"-"*30}\n') elif dsrcAction == 'X': print(f'\n{"-"*30} Reevaluating {"-"*30}\n') else: print(f'\n{"-"*30} Loading {"-"*30}\n') - else: - if not args.createJsonOnly: - print(f'\nTesting {filePath}, CTRL-C to end test at any time...\n') - else: - file_path_json = f'{filePath}.json' - print(f'\nCreating {file_path_json}...\n') - try: - outputFile = open(file_path_json, 'w', encoding='utf-8', newline='') - except IOError as ex: - print(f'\nERROR: Could not create output file {file_path_json}: {ex}') - exitCode = 1 - return # Drop to a single thread for files under 500k if os.path.getsize(filePath) < (100000 if isCompressedFile(filePath) else 500000): @@ -685,31 +691,16 @@ def perform_load(): rowData['DATA_SOURCE'] = sourceDict['DATA_SOURCE'] if 'ENTITY_TYPE' not in rowData and 'ENTITY_TYPE' in sourceDict: rowData['ENTITY_TYPE'] = sourceDict['ENTITY_TYPE'] - if 'LOAD_ID' not in rowData: - rowData['LOAD_ID'] = sourceDict['FILE_NAME'] - - # Update with file defaults - if sourceDict['MAPPING_FILE']: - rowData['_MAPPING_FILE'] = sourceDict['MAPPING_FILE'] if args.testMode: - if '_MAPPING_FILE' not in rowData: - recordList = [rowData] - else: - recordList, errorCount = g2Project.csvMapper(rowData) - if errorCount: - cntBadParse += 1 - recordList = [] - - for rowData in recordList: - mappingResponse = g2Project.mapJsonRecord(rowData) - if mappingResponse[0]: - cntBadUmf += 1 - okToContinue = False - for mappingError in mappingResponse[0]: - print(' WARNING: mapping error in row %s (%s)' % (cntRows, mappingError)) - elif args.createJsonOnly: - outputFile.write(json.dumps(rowData, sort_keys=True) + '\n') + mappingResponse = g2Project.testJsonRecord(rowData, cntRows, sourceDict) + if mappingResponse[0]: + cntBadUmf += 1 + okToContinue = False + + #--only add force a load_id if not in test mode (why do we do this??) + if 'LOAD_ID' not in rowData: + rowData['LOAD_ID'] = sourceDict['FILE_NAME'] # Put the record on the queue if okToContinue: @@ -795,10 +786,6 @@ def perform_load(): print(" Removing temporary file created by S3 download " + filePath) os.remove(filePath) - # Close output file if created json - if args.createJsonOnly: - outputFile.close() - # Remove shuffled file unless run with -snd or prior shuffle detected and not small file/low thread count if not args.shuffleNoDelete \ and not shuf_detected \ @@ -812,7 +799,7 @@ def perform_load(): # Stop processes and threads stopLoaderProcessAndThreads(threadList, workQueue) - # Print load stats if not error or no ctrl-c + # Print load stats if not error or ctrl-c if exitCode in (0, 9): elapsedSecs = time.time() - fileStartTime elapsedMins = round(elapsedSecs / 60, 1) @@ -849,21 +836,6 @@ def perform_load(): Time paused in governor(s): {round(time_governing.value / 60, 1)} mins Records per second: {fileTps} ''')) - statPack = g2Project.getStatPack() - if statPack['FEATURES']: - print(' Features:') - for stat in statPack['FEATURES']: - print((' ' + stat['FEATURE'] + ' ' + '.' * 25)[0:25] + ' ' + (str(stat['COUNT']) + ' ' * 12)[0:12] + ' (' + str(stat['PERCENT']) + '%)') - if statPack['UNMAPPED']: - print(' Unmapped:') - for stat in statPack['UNMAPPED']: - print((' ' + stat['ATTRIBUTE'] + ' ' + '.' * 25)[0:25] + ' ' + (str(stat['COUNT']) + ' ' * 12)[0:12] + ' (' + str(stat['PERCENT']) + '%)') - - # Governor called for each source - try: - source_governor.govern() - except Exception as err: - shutdown(f'\nERROR: Calling per source governor: {err}') # Don't process next source file if errors if exitCode: @@ -896,6 +868,29 @@ def perform_load(): ./G2Loader.py -R {'-c ' + str(iniFileName) if args.iniFile else ''} ''')) + + if args.testMode: + + report = g2Project.getTestResults('F') + report += '\nPress (s)ave to file or (q)uit when reviewing is complete\n' + + less = subprocess.Popen(["less", "-FMXSR"], stdin=subprocess.PIPE) + + # Less returns BrokenPipe if hitting q to exit earlier than end of report + try: + less.stdin.write(report.encode('utf-8')) + except IOError: + pass + + less.stdin.close() + less.wait() + + # Governor called for each source + try: + source_governor.govern() + except Exception as err: + shutdown(f'\nERROR: Calling per source governor: {err}') + return exitCode @@ -1001,6 +996,13 @@ def addEntityType(g2ConfigModule, configDoc, entityType, configuredDatasourcesOn return returnCode +def getInitialG2Config_processWrapper(returnQueue, g2module_params, g2ConfigJson): + result = getInitialG2Config(g2module_params, g2ConfigJson) + # Return values are put in a queue + returnQueue.put(g2ConfigJson) + returnQueue.put(result) + + def getInitialG2Config(g2module_params, g2ConfigJson): # Get the configuration from the ini parms, this is deprecated and G2Health reports this @@ -1031,10 +1033,18 @@ def getInitialG2Config(g2module_params, g2ConfigJson): g2ConfigJson[::] = b'' g2ConfigJson += defaultConfigDoc g2ConfigMgr.destroy() + del g2ConfigMgr return True +def enhanceG2Config_processWrapper(returnQueue, g2Project, g2module_params, g2ConfigJson, configuredDatasourcesOnly): + result = enhanceG2Config(g2Project, g2module_params, g2ConfigJson, configuredDatasourcesOnly) + # Return values are put in a queue + returnQueue.put(g2ConfigJson) + returnQueue.put(result) + + def enhanceG2Config(g2Project, g2module_params, g2ConfigJson, configuredDatasourcesOnly): # verify that we have the needed entity type @@ -1085,8 +1095,10 @@ def enhanceG2Config(g2Project, g2module_params, g2ConfigJson, configuredDatasour print("Error: Failed to set new config as default") return False g2ConfigMgr.destroy() + del g2ConfigMgr g2Config.destroy() + del g2Config return True @@ -1153,6 +1165,7 @@ def sendToG2(threadId_, workQueue_, numThreads_, debugTrace, threadStop, workloa try: g2_engine.destroy() + del g2_engine except Exception: pass @@ -1179,52 +1192,38 @@ def g2Thread(threadId_, workQueue_, g2Engine_, threadStop, workloadStats, dsrcAc row = None continue - # Perform mapping if necessary + dataSource = recordID = '' + # Only umf is not a dict (csv and json are) if type(row) == dict: - if '_MAPPING_FILE' not in row: - rowList = [row] - else: - rowList, errorCount = G2Project.csvMapper(row) - if errorCount: - rowList = [] - else: - rowList = [row] - - # Call g2engine - for row in rowList: - - dataSource = recordID = '' - # Only umf is not a dict (csv and json are) - if type(row) == dict: - if dsrcAction in ('D', 'X'): # strip deletes and reprocess (x) messages down to key only - newRow = {} - dataSource = newRow['DATA_SOURCE'] = row['DATA_SOURCE'] - newRow['DSRC_ACTION'] = dsrcAction - if 'ENTITY_TYPE' in row: - newRow['ENTITY_TYPE'] = row['ENTITY_TYPE'] - if 'ENTITY_KEY' in row: - newRow['ENTITY_KEY'] = row['ENTITY_KEY'] - if 'RECORD_ID' in row: - recordID = newRow['RECORD_ID'] = row['RECORD_ID'] - newRow['DSRC_ACTION'] = dsrcAction - #print ('-'* 25) - #print(json.dumps(newRow, indent=4)) - row = newRow - row = json.dumps(row, sort_keys=True) - - numProcessed += 1 - if (workloadStats and (numProcessed % (args.max_threads_per_process * args.loadOutputFrequency)) == 0): - dump_workload_stats(g2Engine_) + if dsrcAction in ('D', 'X'): # strip deletes and reprocess (x) messages down to key only + newRow = {} + dataSource = newRow['DATA_SOURCE'] = row['DATA_SOURCE'] + newRow['DSRC_ACTION'] = dsrcAction + if 'ENTITY_TYPE' in row: + newRow['ENTITY_TYPE'] = row['ENTITY_TYPE'] + if 'ENTITY_KEY' in row: + newRow['ENTITY_KEY'] = row['ENTITY_KEY'] + if 'RECORD_ID' in row: + recordID = newRow['RECORD_ID'] = row['RECORD_ID'] + newRow['DSRC_ACTION'] = dsrcAction + #print ('-'* 25) + #print(json.dumps(newRow, indent=4)) + row = newRow + row = json.dumps(row, sort_keys=True) + + numProcessed += 1 + if (workloadStats and (numProcessed % (args.max_threads_per_process * args.loadOutputFrequency)) == 0): + dump_workload_stats(g2Engine_) - try: - # Temp fix for distinguising between X messages that are either JSON or UMF. - row_is_json = type(row) == dict + try: + # Temp fix for distinguising between X messages that are either JSON or UMF. + row_is_json = type(row) == dict # <-ExampleQ-Remove-> - if row_is_json and dsrcAction == 'X': - g2Engine_.reevaluateRecord(dataSource, recordID, 0) - else: - g2Engine_.process(row) + if row_is_json and dsrcAction == 'X': + g2Engine_.reevaluateRecord(dataSource, recordID, 0) + else: + g2Engine_.process(row) # <-ExampleQ-Remove-> # <-ExampleQ-> response = bytearray() @@ -1242,18 +1241,18 @@ def g2Thread(threadId_, workQueue_, g2Engine_, threadStop, workloadStats, dsrcAc # <-ExampleQ-> if message != '': # <-ExampleQ-> channel.basic_publish(exchange='', routing_key=args.queueName, body=message) - except G2ModuleLicenseException as ex: - print('ERROR: G2Engine licensing error!') - print(f' {ex}') - with threadStop.get_lock(): - threadStop.value = 1 - return - except G2ModuleException as ex: - print(f'ERROR: {ex}') - print(f' {row}') - except Exception as ex: - print(f'ERROR: {ex}') - print(f' {row}') + except G2ModuleLicenseException as ex: + print('ERROR: G2Engine licensing error!') + print(f' {ex}') + with threadStop.get_lock(): + threadStop.value = 1 + return + except G2ModuleException as ex: + print(f'ERROR: {ex}') + print(f' {row}') + except Exception as ex: + print(f'ERROR: {ex}') + print(f' {row}') return @@ -1290,7 +1289,6 @@ def time_now(add_secs=False): return datetime.now().strftime(fmt_string).lower() - def signal_int(signal, frame): ''' Signal interupt handler ''' @@ -1422,7 +1420,6 @@ def governor_cleanup(): g2load_parser.add_argument('-n', '--noRedo', dest='noRedo', action='store_true', default=False, help='disable redo processing') g2load_parser.add_argument('-i', '--redoModeInterval', dest='redoModeInterval', type=int, default=60, help='time in secs to wait between redo processing checks, only used in redo mode') g2load_parser.add_argument('-k', '--knownDatasourcesOnly', dest='configuredDatasourcesOnly', action='store_true', default=False, help='only accepts configured and known data sources') - g2load_parser.add_argument('-j', '--createJsonOnly', dest='createJsonOnly', action='store_true', default=False, help='only create JSON files from mapped CSV files') g2load_parser.add_argument('-mtp', '--maxThreadsPerProcess', dest='max_threads_per_process', default=16, type=int, help='maximum threads per process, default=%(default)s') g2load_parser.add_argument('-g', '--governor', dest='governor', default=None, help='user supplied governor to load and call during processing', nargs=1) g2load_parser.add_argument('-gpd', '--governorDisable', dest='governorDisable', action='store_true', default=False, help='disable default Postgres governor, when repository is Postgres') @@ -1559,11 +1556,26 @@ def governor_cleanup(): ''')) == "YESPURGE": sys.exit(0) + # test mode settings + if args.testMode: + defaultThreadCount = 1 + args.loadOutputFrequency = 10000 if args.loadOutputFrequency == 1000 else args.loadOutputFrequency + # Check resources and acquire num threads - defaultThreadCount = check_resources_and_startup(thread_count=args.thread_count, - doPurge=(args.purgeFirst or args.forcePurge) and not (args.testMode or args.redoMode), - doLicense=True, - ) + else: + tempQueue = Queue() + checkResourcesProcess = Process(target=check_resources_and_startup, + args=(tempQueue, + args.thread_count, + (args.purgeFirst or args.forcePurge) and not (args.testMode or args.redoMode), + True)) + checkResourcesProcess.start() + checkResourcesProcess.join() + defaultThreadCount = tempQueue.get() + + # Exit if checkResourcesProcess failed to start an engine + if defaultThreadCount == -1: + sys.exit(1) # -w has been deprecated and the default is now output workload stats, noWorkloadStats is checked in place of # workloadStats. -w will be removed in future updates. @@ -1578,9 +1590,6 @@ def governor_cleanup(): args.shuffleNoDelete = True time.sleep(5) - if args.createJsonOnly: - args.testMode = True - # Setup the governor(s) governor_setup() diff --git a/g2/python/G2Project.py b/g2/python/G2Project.py index dc87429..cdb98fd 100644 --- a/g2/python/G2Project.py +++ b/g2/python/G2Project.py @@ -1,51 +1,49 @@ #! /usr/bin/env python3 -#### import csv import fnmatch import glob +import io import json import optparse import os import sys import textwrap +from contextlib import redirect_stdout from operator import itemgetter from CompressedFile import (fileRowParser, isCompressedFile, openPossiblyCompressedFile) -#### from G2Exception import (G2InvalidFileTypeContentsException, G2UnsupportedFileTypeException) from G2S3 import G2S3 -#### try: -#### from dateutil.parser import parse as dateParser -#### except Exception: -#### pass - -try: - from nameparser import HumanName as nameParser -except Exception: - pass #====================== class G2Project: #====================== #---------------------------------------- - def __init__(self, g2ConfigTables, projectFileName = None, projectFileUri = None, tempFolderPath = None): + def __init__(self, g2ConfigTables, dsrcAction, projectFileName=None, projectFileUri=None, tempFolderPath=None): """ open and validate a g2 generic configuration and project file """ self.projectSourceList = [] self.mappingFiles = {} - self.attributeDict = g2ConfigTables.loadConfig('CFG_ATTR') self.dataSourceDict = g2ConfigTables.loadConfig('CFG_DSRC') self.entityTypeDict = g2ConfigTables.loadConfig('CFG_ETYPE') + self.featureDict = g2ConfigTables.loadConfig('CFG_FTYPE') + self.attributeDict = g2ConfigTables.loadConfig('CFG_ATTR') + + self.f1Features = [] + for feature in self.featureDict: + if self.featureDict[feature]['FTYPE_FREQ'] == 'F1': + self.f1Features.append(feature) self.success = True self.mappingCache = {} self.tempFolderPath = tempFolderPath + self.dsrcAction = dsrcAction self.clearStatPack() - self.loadAttributes() + if self.success and projectFileName: self.loadProjectFile(projectFileName) elif self.success and projectFileUri: @@ -57,28 +55,6 @@ def __init__(self, g2ConfigTables, projectFileName = None, projectFileUri = None #--mapping functions #----------------------------- - #---------------------------------------- - def loadAttributes(self): - ''' creates a feature/element structure out of the flat attributes in the mapping file ''' - - #--create the list of elements per feature so that we can validate the mappings - attributeDict = self.attributeDict - self.featureDict = {} - for configAttribute in attributeDict: - if attributeDict[configAttribute]['FTYPE_CODE']: - if attributeDict[configAttribute]['FTYPE_CODE'] not in self.featureDict: - self.featureDict[attributeDict[configAttribute]['FTYPE_CODE']] = {} - self.featureDict[attributeDict[configAttribute]['FTYPE_CODE']]['FEATURE_ORDER'] = attributeDict[configAttribute]['ATTR_ID'] - self.featureDict[attributeDict[configAttribute]['FTYPE_CODE']]['ATTR_CLASS'] = attributeDict[configAttribute]['ATTR_CLASS'] - self.featureDict[attributeDict[configAttribute]['FTYPE_CODE']]['ELEMENT_LIST'] = [] - if attributeDict[configAttribute]['ATTR_ID'] < self.featureDict[attributeDict[configAttribute]['FTYPE_CODE']]['FEATURE_ORDER']: - self.featureDict[attributeDict[configAttribute]['FTYPE_CODE']]['FEATURE_ORDER'] = attributeDict[configAttribute]['ATTR_ID'] - - #--reclass dict no longer valid - self.reclassDict = {} - - return - #---------------------------------------- def lookupAttribute(self, attrName): ''' determine if a valid config attribute ''' @@ -129,54 +105,20 @@ def mapAttribute(self, attrName, attrValue): ''' places mapped column values into a feature structure ''' #--strip spaces and ensure a string - if attrValue: - if type(attrValue) in (str, str): - attrValue = attrValue.strip() - #if attrValue.upper() == 'NULL' or self.containsOnly(attrValue, '.,-?'): - # attrValue = None - else: - attrValue = str(attrValue) + if not str(attrValue).strip(): + return None - #--if still a column value - if attrValue: - attrMapping = self.lookupAttribute(attrName) - attrMapping['ATTR_NAME'] = attrName - attrMapping['ATTR_VALUE'] = attrValue - else: - attrMapping = None + attrMapping = self.lookupAttribute(attrName) + attrMapping['ATTR_NAME'] = attrName + attrMapping['ATTR_VALUE'] = attrValue return attrMapping - def listHasLowerCaseKeys(self,listCollection): - hasLowerCaseKeys = False - for item in listCollection: - if type(item) is dict: - hasLowerCaseKeys = hasLowerCaseKeys or self.dictHasLowerCaseKeys(item) - elif type(item) is list: - hasLowerCaseKeys = hasLowerCaseKeys or self.listHasLowerCaseKeys(item) - return hasLowerCaseKeys - - def dictHasLowerCaseKeys(self,dictCollection): - hasLowerCaseKeys = False - for dictKey, dictValue in dictCollection.items(): - for c in dictKey: - if c.islower(): - hasLowerCaseKeys = True - if type(dictValue) is dict: - hasLowerCaseKeys = hasLowerCaseKeys or self.dictHasLowerCaseKeys(dictValue) - elif type(dictValue) is list: - hasLowerCaseKeys = hasLowerCaseKeys or self.listHasLowerCaseKeys(dictValue) - return hasLowerCaseKeys - - def recordHasLowerCaseKeys(self,jsonRecord): - return self.dictHasLowerCaseKeys(jsonRecord) - #---------------------------------------- - def mapJsonRecord(self, jsonDict): - '''checks for mapping errors''' + def testJsonRecord(self, jsonDict, rowNum, sourceDict): mappingErrors = [] mappedAttributeList = [] - valuesByClass = {} + mappedFeatures = {} entityName = None self.recordCount += 1 @@ -187,7 +129,7 @@ def mapJsonRecord(self, jsonDict): for childRow in jsonDict[columnName]: childRowNum += 1 if type(childRow) != dict: - mappingErrors.append('expected {records} in list under %s' % columnName) + mappingErrors.append('expected records in list under %s' % columnName) break else: for childColumn in childRow: @@ -197,19 +139,20 @@ def mapJsonRecord(self, jsonDict): mappedAttribute['ATTR_LEVEL'] = columnName + '#' + str(childRowNum) mappedAttributeList.append(mappedAttribute) else: - mappingErrors.append('unexpected {records} under %s of %s' % (childColumn, columnName)) + mappingErrors.append('unexpected records under %s of %s' % (childColumn, columnName)) break elif type(jsonDict[columnName]) != dict: #--safeguard - columnValue = jsonDict[columnName] mappedAttribute = self.mapAttribute(columnName, jsonDict[columnName]) if mappedAttribute: mappedAttribute['ATTR_LEVEL'] = 'ROOT' mappedAttributeList.append(mappedAttribute) else: - mappingErrors.append('unexpected {records} under %s' % columnName) + mappingErrors.append('unexpected records under %s' % columnName) - if not mappingErrors: + if mappingErrors: + self.updateStatPack('ERROR', 'Invalid JSON structure', 'row: ' + str(rowNum)) + else: #--create a feature key to group elements that go together elementKeyVersions = {} @@ -237,22 +180,33 @@ def mapJsonRecord(self, jsonDict): i = 0 while i < mappedAttributeListLength: if mappedAttributeList[i]['FEATURE_KEY']: + featureKey = mappedAttributeList[i]['FEATURE_KEY'] ftypeClass = mappedAttributeList[i]['ATTR_CLASS'] ftypeCode = mappedAttributeList[i]['FTYPE_CODE'] utypeCode = mappedAttributeList[i]['UTYPE_CODE'] featureDesc = '' + attrList = [] completeFeature = False while i < mappedAttributeListLength and mappedAttributeList[i]['FEATURE_KEY'] == featureKey: + attrCode = mappedAttributeList[i]['ATTR_CODE'] + attrValue = mappedAttributeList[i]['ATTR_VALUE'] + if attrCode != ftypeCode: + self.updateStatPack('MAPPED', f'{ftypeCode}|{utypeCode}|{attrCode}', attrValue) if mappedAttributeList[i]['FELEM_CODE'] == 'USAGE_TYPE': utypeCode = mappedAttributeList[i]['ATTR_VALUE'] elif mappedAttributeList[i]['FELEM_CODE'].upper() not in ('USED_FROM_DT', 'USED_THRU_DT'): - featureDesc += ('' if len(featureDesc) == 0 else ' ') + mappedAttributeList[i]['ATTR_VALUE'] - if mappedAttributeList[i]['FELEM_REQ'].upper() in ('YES', 'ANY'): - completeFeature = True + if len(str(mappedAttributeList[i]['ATTR_VALUE'])) if mappedAttributeList[i]['ATTR_VALUE'] != None else 0: #--can't count null values + attrList.append(attrCode) + featureDesc += ' ' + mappedAttributeList[i]['ATTR_VALUE'] + if mappedAttributeList[i]['FELEM_REQ'].upper() in ('YES', 'ANY'): + completeFeature = True i += 1 - if completeFeature: + if not completeFeature: + self.updateStatPack('INFO', f'Incomplete {ftypeCode}', f'row {self.recordCount}') + else: + self.updateStatPack('MAPPED', f'{ftypeCode}|{utypeCode}|n/a', featureDesc.strip()) featureCount += 1 #--update mapping stats @@ -262,19 +216,17 @@ def mapJsonRecord(self, jsonDict): else: self.featureStats[statCode] = 1 - #--yse first name encountered as the entity description + + #--use first name encountered as the entity description if ftypeCode == 'NAME' and not entityName: entityName = featureDesc - #--update values by class - if utypeCode: - featureDesc = utypeCode +': ' + featureDesc - if ftypeCode not in ('NAME', 'ADDRESS', 'PHONE'): - featureDesc = ftypeCode +': ' + featureDesc - if ftypeClass in valuesByClass: - valuesByClass[ftypeClass] += '\n' + featureDesc + #--capture feature stats for validation + validationData = {'USAGE_TYPE': utypeCode, 'ATTR_LIST': attrList} + if ftypeCode not in mappedFeatures: + mappedFeatures[ftypeCode] = [validationData] else: - valuesByClass[ftypeClass] = featureDesc + mappedFeatures[ftypeCode].append(validationData) else: #--this is an unmapped attribute @@ -287,118 +239,189 @@ def mapJsonRecord(self, jsonDict): else: self.unmappedStats[statCode] = 1 - #--update values by class - attrClass = mappedAttributeList[i]['ATTR_CLASS'] - attrDesc = mappedAttributeList[i]['ATTR_NAME'] +': ' + mappedAttributeList[i]['ATTR_VALUE'] - if attrClass in valuesByClass: - valuesByClass[attrClass] += '\n' + attrDesc - else: - valuesByClass[attrClass] = attrDesc + attrCode = mappedAttributeList[i]['ATTR_NAME'] + attrValue = mappedAttributeList[i]['ATTR_VALUE'] + self.updateStatPack('UNMAPPED', attrCode, attrValue) elif mappedAttributeList[i]['ATTR_CLASS'] == 'OBSERVATION': - pass #--no need as not creating umf + attrCode = mappedAttributeList[i]['ATTR_NAME'] + attrValue = mappedAttributeList[i]['ATTR_VALUE'] + self.updateStatPack('MAPPED', 'n/a||' + attrCode, attrValue) #--"n/a||"" to give same structure as an actual feature i += 1 - if featureCount == 0: - mappingErrors.append('no features mapped') - - return [mappingErrors, mappedAttributeList, valuesByClass, entityName] + #--errors and warnings + messageList = [] + if self.dsrcAction == 'A' and featureCount == 0: + messageList.append(['ERROR', 'No features mapped']) + + #--required missing values + if 'DATA_SOURCE' not in jsonDict and 'DATA_SOURCE' not in sourceDict: + messageList.append(['ERROR', 'Data source missing']) + #--this next condition is confusing because if they specified data source for the file (sourceDict) it will be added automatically + #--so if the data source was not specified for the file its in the json record and needs to be validated + elif 'DATA_SOURCE' not in sourceDict and jsonDict['DATA_SOURCE'].upper() not in self.dataSourceDict: + messageList.append(['ERROR', 'Invalid data source: ' + jsonDict['DATA_SOURCE'].upper()]) + if 'ENTITY_TYPE' in jsonDict and jsonDict['ENTITY_TYPE'].upper() not in self.entityTypeDict: + messageList.append(['ERROR', 'Invalid entity type: ' + jsonDict['ENTITY_TYPE'].upper()]) + + #--record_id + if 'RECORD_ID' not in jsonDict: + messageList.append(['INFO', 'Record ID is missing']) + record_id = '' + else: + record_id = jsonDict['RECORD_ID'] - #---------------------------------------- - def validateJsonMessage(self, msg): - ''' validates a json message and return the mappings ''' - jsonErrors = [] - jsonMappings = {} - if type(msg) == dict: - jsonDict = msg - else: - try: jsonDict = json.loads(msg, encoding="utf-8") - except: - jsonErrors.append('ERROR: could not parse as json') - jsonDict = None - - if not jsonErrors: - ##print '-' * 50 - for columnName in jsonDict: - ##print columnName+'-'*10, type(jsonDict[columnName]) - if type(jsonDict[columnName]) == dict: - jsonErrors.append('ERROR: single value expected: %s : %s' % (columnName, json.dumps(jsonDict[columnName]))) - elif type(jsonDict[columnName]) == list: - childRowNum = 0 - for childRow in jsonDict[columnName]: - childRowNum += 1 - if type(childRow) != dict: - jsonErrors.append('ERROR: dict expected: %s : %s' % (columnName, json.dumps(jsonDict[columnName]))) - break - else: - for childColumn in childRow: - ##print str(childRowNum) + ' ' + childColumn + '-'*8, type(childColumn) - if type(childColumn) in (dict, list): - jsonErrors.append('ERROR: single value expected: %s : %s' % (columnName, json.dumps(jsonDict[columnName]))) - break - elif childColumn not in jsonMappings: - jsonMappings[childColumn] = self.lookupAttribute(childColumn) + #--name warnings + if 'NAME' not in mappedFeatures: + messageList.append(['INFO', 'Missing Name']) + else: + crossAttrList = [] + for validationData in mappedFeatures['NAME']: + crossAttrList += validationData['ATTR_LIST'] + if 'NAME_FULL' in validationData['ATTR_LIST'] and any(item in validationData['ATTR_LIST'] for item in ['NAME_FIRST', 'NAME_LAST', 'NAME_ORG']): + messageList.append(['INFO', 'Full name should be mapped alone']) + elif 'NAME_LAST' in validationData['ATTR_LIST'] and 'NAME_FIRST' not in validationData['ATTR_LIST']: + messageList.append(['INFO', 'Last name without first name']) + elif 'NAME_FIRST' in validationData['ATTR_LIST'] and 'NAME_LAST' not in validationData['ATTR_LIST']: + messageList.append(['INFO', 'First name without last name']) + if 'NAME_ORG' in crossAttrList and any(item in crossAttrList for item in ['NAME_FIRST', 'NAME_LAST']): + messageList.append(['WARNING', 'Organization and person names on same record']) + + #--address warnings + if 'ADDRESS' in mappedFeatures: + for validationData in mappedFeatures['ADDRESS']: + if 'ADDR_FULL' in validationData['ATTR_LIST']: + if any(item in validationData['ATTR_LIST'] for item in ['ADDR_LINE1', 'ADDR_CITY', 'ADDR_STATE', 'ADDR_POSTAL_CODE', 'ADDR_COUNTRY' ]): + messageList.append(['INFO', 'Full address should be mapped alone']) + else: + if 'ADDR_LINE1' not in validationData['ATTR_LIST']: + messageList.append(['INFO', 'Address line1 is missing']) + if 'ADDR_CITY' not in validationData['ATTR_LIST']: + messageList.append(['INFO', 'Address city is missing']) + if 'ADDR_POSTAL_CODE' not in validationData['ATTR_LIST']: + messageList.append(['INFO', 'Address postal code is missing']) + + #--other warnings + if 'OTHER_ID' in mappedFeatures: + if len(mappedFeatures['OTHER_ID']) > 1: + messageList.append(['INFO', 'Multiple other_ids mapped']) + else: + messageList.append(['INFO', 'Use of other_id feature']) - elif columnName not in jsonMappings: - jsonMappings[columnName] = self.lookupAttribute(columnName) + for message in messageList: + self.updateStatPack(message[0], message[1], 'row: ' + str(rowNum) + (', record_id: ' + record_id if record_id else '')) + mappingErrors.append(message) - return jsonErrors, jsonDict, jsonMappings + return [mappingErrors, mappedAttributeList, entityName] #--------------------------------------- def clearStatPack(self): ''' clear the statistics on demand ''' self.recordCount = 0 + self.statPack = {} + self.featureStats = {} self.unmappedStats = {} return + #---------------------------------------- + def updateStatPack(self, cat1, cat2, value=None): + + if cat1 not in self.statPack: + self.statPack[cat1] = {} + if cat2 not in self.statPack[cat1]: + self.statPack[cat1][cat2] = {'COUNT': 1, 'VALUES': {}} + else: + self.statPack[cat1][cat2]['COUNT'] += 1 + + if value: + if value not in self.statPack[cat1][cat2]['VALUES']: + self.statPack[cat1][cat2]['VALUES'][value] = 1 + else: + self.statPack[cat1][cat2]['VALUES'][value] += 1 + return + #--------------------------------------- def getStatPack(self): - - ''' return the statistics for each feature ''' statPack = {} + fileWarnings = [] + for cat1 in self.statPack: + itemList = [] + for cat2 in self.statPack[cat1]: + reclass_warning = False + itemDict = {} + if cat1 == 'MAPPED': + cat2parts = cat2.split('|') + itemDict['feature'] = cat2parts[0] + itemDict['label'] = cat2parts[1] + itemDict['attribute'] = cat2parts[2] + itemDict['featId'] = self.featureDict[cat2parts[0]]['ID'] if cat2parts[0] != 'n/a' else 0 + itemDict['attrId'] = self.attributeDict[cat2parts[2]]['ATTR_ID'] if cat2parts[2] != 'n/a' else 0 + if not itemDict['attrId']: + itemDict['description'] = itemDict['feature'] + (' - ' + itemDict['label'] if itemDict['label'] else '') + elif not itemDict['featId']: + itemDict['description'] = itemDict['attribute'] + else: + itemDict['description'] = ' ' + itemDict['attribute'].lower() - featureStats = [] - for feature in self.featureStats: - featureStat = {} - featureStat['FEATURE'] = feature - if '-' in feature: - featureSplit = feature.split('-') - featureStat['FTYPE_CODE'] = featureSplit[0] - featureStat['UTYPE_CODE'] = featureSplit[1] - else: - featureStat['FTYPE_CODE'] = feature - featureStat['UTYPE_CODE'] = '' - featureStat['FEATURE_ORDER'] = self.featureDict[featureStat['FTYPE_CODE']]['FEATURE_ORDER'] - featureStat['COUNT'] = self.featureStats[feature] - if self.recordCount == 0: - featureStat['PERCENT'] = 0 - else: - featureStat['PERCENT'] = round((float(featureStat['COUNT']) / self.recordCount) * 100,2) - featureStats.append(featureStat) - statPack['FEATURES'] = sorted(featureStats, key=itemgetter('FEATURE_ORDER', 'UTYPE_CODE')) - - unmappedStats = [] - for attribute in self.unmappedStats: - unmappedStat = {} - unmappedStat['ATTRIBUTE'] = attribute - unmappedStat['COUNT'] = self.unmappedStats[attribute] - if self.recordCount == 0: - unmappedStat['PERCENT'] = 0 - else: - unmappedStat['PERCENT'] = round((float(unmappedStat['COUNT']) / self.recordCount) * 100,2) - unmappedStats.append(unmappedStat) - statPack['UNMAPPED'] = sorted(unmappedStats, key=itemgetter('ATTRIBUTE')) + elif cat1 == 'UNMAPPED': + itemDict['attribute'] = cat2 + else: + itemDict['type'] = cat1 + itemDict['message'] = cat2 + + itemDict['recordCount'] = self.statPack[cat1][cat2]['COUNT'] + itemDict['recordPercent'] = self.statPack[cat1][cat2]['COUNT'] / self.recordCount if self.recordCount else 0 + if cat1 in ('MAPPED', 'UNMAPPED'): + itemDict['uniqueCount'] = len(self.statPack[cat1][cat2]['VALUES']) + itemDict['uniquePercent'] = itemDict['uniqueCount'] / itemDict['recordCount'] if itemDict['recordCount'] else 0 + + #--feature warnings (not statistically relevant on small amounts of data) + if cat1 == 'MAPPED' and itemDict['attrId'] == 0 and itemDict['recordCount'] >= 1000: + itemDict['frequency'] = self.featureDict[cat2parts[0]]['FTYPE_FREQ'] + if itemDict['frequency'] == 'F1' and itemDict['uniquePercent'] < .8: + itemDict['warning'] = True + msg = itemDict['feature'] + ' is only ' + str(round(itemDict['uniquePercent'] * 100, 0)) +'% unique' + fileWarnings.append({'type': 'WARNING', 'message': msg, 'recordCount': itemDict['uniqueCount'], 'recordPercent': itemDict['uniquePercent']}) + if itemDict['frequency'] == 'NAME' and itemDict['uniquePercent'] < .7: + itemDict['warning'] = True + msg = itemDict['feature'] + ' is only ' + str(round(itemDict['uniquePercent'] * 100, 0)) +'% unique' + fileWarnings.append({'type': 'WARNING', 'message': msg, 'recordCount': itemDict['uniqueCount'], 'recordPercent': itemDict['uniquePercent']}) + if itemDict['frequency'] == 'FF' and itemDict['uniquePercent'] < .7: + itemDict['warning'] = True + msg = itemDict['feature'] + ' is only ' + str(round(itemDict['uniquePercent'] * 100, 0)) +'% unique' + fileWarnings.append({'type': 'WARNING', 'message': msg, 'recordCount': itemDict['uniqueCount'], 'recordPercent': itemDict['uniquePercent']}) + + #--reclass prevalent informational messages to warnings + elif cat1 == 'INFO' and itemDict['recordPercent'] >= .5: + reclass_warning = True + + itemDict['topValues'] = [] + for value in sorted(self.statPack[cat1][cat2]['VALUES'].items(), key=lambda x: x[1], reverse=True): + itemDict['topValues'].append('%s (%s)' % value) + if len(itemDict['topValues']) == 10: + break - return statPack + if reclass_warning: + fileWarnings.append(itemDict) + else: + itemList.append(itemDict) - #--------------------------------------- - def featureToJson(self, featureList): - ''' turns database feature (felem_values) strings back into json attributes ''' - jsonString = None + if cat1 == 'MAPPED': + statPack[cat1] = sorted(itemList, key=lambda x: (x['featId'], x['label'], x['attrId'])) + elif cat1 == 'UNMAPPED': + statPack[cat1] = sorted(itemList, key=lambda x: x['attribute']) + else: + statPack[cat1] = sorted(itemList, key=lambda x: x['recordCount'], reverse = True) - return jsonString + if fileWarnings and 'WARNING' not in statPack: + statPack['WARNING'] = [] + for itemDict in fileWarnings: + itemDict['type'] = 'WARNING' + statPack['WARNING'].insert(0, itemDict) + + return statPack #----------------------------- #--project functions @@ -531,18 +554,6 @@ def loadProjectFile(self, projectFileName): return - #---------------------------------------- - def dictKeysUpper(self, in_dict): - if type(in_dict) is dict: - out_dict = {} - for key, item in in_dict.items(): - out_dict[key.upper()] = self.dictKeysUpper(item) - return out_dict - elif type(in_dict) is list: - return [self.dictKeysUpper(obj) for obj in in_dict] - else: - return in_dict - #---------------------------------------- def loadJsonProject(self): ''' validates and loads a json project file into memory ''' @@ -554,9 +565,9 @@ def loadJsonProject(self): else: projectData = self.dictKeysUpper(projectData) if type(projectData) == list: - projectData = {"DATA_SOURCES": projectData} + projectData = {'DATA_SOURCES': projectData} - if "DATA_SOURCES" in projectData: + if 'DATA_SOURCES' in projectData: sourceRow = 0 for sourceDict in projectData['DATA_SOURCES']: sourceRow += 1 @@ -630,8 +641,6 @@ def prepareSourceFiles(self): if 'DATA_SOURCE' in sourceDict: sourceDict['DATA_SOURCE'] = sourceDict['DATA_SOURCE'].strip().upper() - if 'ENTITY_TYPE' not in sourceDict: - sourceDict['ENTITY_TYPE'] = 'GENERIC' if 'FILE_FORMAT' not in sourceDict: fileName, fileExtension = os.path.splitext(sourceDict['FILE_NAME']) @@ -727,44 +736,21 @@ def prepareSourceFiles(self): if sourceDict['FILE_FORMAT'] == 'UMF': if not (rowData.upper().startswith('')): - print(' WARNING: invalid UMF in row %s (%s)' % (rowCnt, rowData[0:50])) + print('WARNING: invalid UMF in row %s (%s)' % (rowCnt, rowData[0:50])) + badCnt += 1 + else: #--test json or csv record + mappingResponse = self.testJsonRecord(rowData, rowCnt, sourceDict) + errors = False + for mappingError in mappingResponse[0]: + if mappingError[0] == 'ERROR': + errors = True + print(f' {mappingError[0]}: Row {rowCnt} - {mappingError[1]}') + if errors: badCnt += 1 - else: #--json or csv - - #--perform csv mapping if needed - if not sourceDict['MAPPING_FILE']: - recordList = [rowData] - else: - rowData['_MAPPING_FILE'] = sourceDict['MAPPING_FILE'] - recordList, errorCount = self.csvMapper(rowData) - if errorCount: - badCnt += 1 - recordList = [] - - #--ensure output(s) have mapped features - for rowData in recordList: - if 'DATA_SOURCE' not in rowData and 'DATA_SOURCE' not in sourceDict: - print(' WARNING: data source missing in row %s and not specified at the file level' % rowCnt) - badCnt += 1 - elif 'DATA_SOURCE' in rowData and rowData['DATA_SOURCE'].upper() not in self.dataSourceDict: - print(' WARNING: invalid data_source in row %s (%s)' % (rowCnt, rowData['DATA_SOURCE'])) - badCnt += 1 - elif 'ENTITY_TYPE' in rowData and rowData['ENTITY_TYPE'].upper() not in self.entityTypeDict: - print(' WARNING: invalid entity_type in row %s (%s)' % (rowCnt, rowData['ENTITY_TYPE'])) - badCnt += 1 - elif 'DSRC_ACTION' in rowData and rowData['DSRC_ACTION'].upper() == 'X': - pass - else: - #--other mapping errors - mappingResponse = self.mapJsonRecord(rowData) - if mappingResponse[0]: - badCnt += 1 - for mappingError in mappingResponse[0]: - print(' WARNING: mapping error in row %s (%s)' % (rowCnt, mappingError)) #--fails if too many bad records (more than 10 of 100 or all bad) if badCnt >= 10 or badCnt == rowCnt: - print(' ERROR: Pre-test failed %s bad records in first %s' % (badCnt, rowCnt)) + print(f'\nERROR: Pre-test failed {badCnt} bad records in first {rowCnt}') self.success = False fileReader.close() @@ -775,145 +761,114 @@ def prepareSourceFiles(self): return #---------------------------------------- - def csvMapper(self, rowData): - outputRows = [] - - #--clean garbage values - for key in rowData: - try: - if rowData[key].upper() in ('NULL', 'NONE', 'N/A', '\\N'): - rowData[key] = '' - except Exception: - pass - - mappingErrors = 0 - csvMap = self.mappingFiles[rowData['_MAPPING_FILE']] - - if 'CALCULATIONS' in csvMap: - - if type(csvMap['CALCULATIONS']) == dict: - for newField in csvMap['CALCULATIONS']: - try: rowData[newField] = eval(csvMap['CALCULATIONS'][newField]) - except Exception as e: - print(' error: %s [%s]' % (newField, e)) - mappingErrors += 1 - - elif type(csvMap['CALCULATIONS']) == list: - for calcDict in csvMap['CALCULATIONS']: - try: rowData[calcDict['NAME']] = eval(calcDict['EXPRESSION']) - except Exception as e: - print(' error: %s [%s]' % (calcDict['NAME'], e)) - mappingErrors += 1 - - #--for each mapping (output record) - for mappingData in csvMap['MAPPINGS']: - mappedData = {} - for columnName in mappingData: - - #--perform the mapping - try: columnValue = mappingData[columnName] % rowData - except: - print(' error: could not map %s' % mappingData[columnName]) - mappingErrors += 1 - columnValue = '' - - #--clear nulls - if not columnValue or columnValue.upper() in ('NONE', 'NULL', '\\N'): - columnValue = '' - - #--dont write empty tags - if columnValue: - mappedData[columnName] = columnValue - - outputRows.append(mappedData) - - return outputRows, mappingErrors - -#-------------------- -#--utility functions -#-------------------- - -#---------------------------------------- -def pause(question = None): - if not question: - v_wait = input("PRESS ENTER TO CONTINUE ... ") - else: - v_wait = input(question) - return v_wait - -#---------------------------------------- -def containsOnly(seq, aset): - ''' Check whether sequence seq contains ONLY items in aset ''' - for c in seq: - if c not in aset: return False - return True - -#---------------------------------------- -def calcNameKey(fullNameStr, keyType): - - if type(fullNameStr) == list: - newStr = '' - for namePart in fullNameStr: - if namePart: - newStr += (' ' + namePart) - fullNameStr = newStr.strip() - - try: - values = [] - parsedName = nameParser(fullNameStr) - if keyType.upper() == 'FULL': - if len(parsedName.last) != 0: - values.append(parsedName.last) - if len(parsedName.first) != 0: - values.append(parsedName.first) - if len(parsedName.middle) != 0: - values.append(parsedName.middle) - else: - if len(parsedName.last) != 0 and len(parsedName.first) != 0 and len(parsedName.middle) != 0: - values = [parsedName.last, parsedName.first] - except: - values = fullNameStr.upper().replace('.',' ').replace(',',' ').split() - - return '|'.join(sorted([x.upper() for x in values])) - -#---------------------------------------- -def calcOrgKey(fullNameStr): - values = fullNameStr.upper().replace('.',' ').replace(',',' ').split() - return ' '.join(values) - -#---------------------------------------- -def compositeKeyBuilder(rowData, keyList): - values = [] - for key in keyList: - if key in rowData and rowData[key]: - values.append(str(rowData[key])) + def dictKeysUpper(self, in_dict): + if type(in_dict) is dict: + out_dict = {} + for key, item in in_dict.items(): + out_dict[key.upper()] = self.dictKeysUpper(item) + return out_dict + elif type(in_dict) is list: + return [self.dictKeysUpper(obj) for obj in in_dict] else: - return '' - return '|'.join(values) - -#---------------------------------------- -if __name__ == "__main__": - - #--running in debug mode - no parameters - if len(sys.argv) == 1: - mappingFileName = './g2Generic.map' - projectFileName = './test/input/project.json' - - #--capture the command line arguments - else: - optParser = optparse.OptionParser() - optParser.add_option('-m', '--mappingFile', dest='mappingFileName', default='', help='the name of a g2 attribute mapping file') - optParser.add_option('-p', '--projectFile', dest='projectFileName', default='', help='the name of a g2 project csv or json file') - (options, args) = optParser.parse_args() - mappingFileName = options.mappingFileName - projectFileName = options.projectFileName - - #--create an instance - myProject = g2Mapper('self', mappingFileName, projectFileName) - if myProject.success: - print('SUCCESS: project ' + projectFileName + ' is valid!') - - #--delete the instance - del myProject - - sys.exit() + return in_dict + + #----------------------------- + #--report functions + #----------------------------- + + def getTestResults(self, reportStyle = 'Full'): + statPack = self.getStatPack() + with io.StringIO() as buf, redirect_stdout(buf): + + print ('\nTEST RESULTS') + + if 'MAPPED' in statPack: + tableInfo = [{'header': 'MAPPED FEATURES', 'width': 40, 'just': '<'}, + {'header': 'UniqueCount', 'width': 0, 'just': '>', 'style': ','}, + {'header': 'UniquePercent', 'width': 0, 'just': '>', 'style': '.1%'}, + {'header': 'RecordCount', 'width': 0, 'just': '>', 'style': ','}, + {'header': 'RecordPercent', 'width': 0, 'just': '>', 'style': '.1%'}] + if reportStyle == 'F': + for i in range(10): + tableInfo.append({'header': f'Top used value {i+1}', 'width': 50}) + tableData = [] + for itemDict in statPack['MAPPED']: + rowData = [itemDict['description'], itemDict['uniqueCount'], itemDict['uniquePercent'], + itemDict['recordCount'], itemDict['recordPercent']] + if reportStyle == 'F': + for i in range(10): + if i < len(itemDict['topValues']): + rowData.append(itemDict['topValues'][i]) + else: + rowData.append('') + tableData.append(rowData) + + print() + self.renderTable(tableInfo, tableData) + + if 'UNMAPPED' in statPack: + tableInfo[0]['header'] = 'UNMAPPED ATTRIBUTES' + tableData = [] + for itemDict in statPack['UNMAPPED']: + rowData = [itemDict['attribute'], itemDict['uniqueCount'], itemDict['uniquePercent'], + itemDict['recordCount'], itemDict['recordPercent']] + if reportStyle == 'F': + for i in range(10): + if i < len(itemDict['topValues']): + rowData.append(itemDict['topValues'][i]) + else: + rowData.append('') + tableData.append(rowData) + + print() + self.renderTable(tableInfo, tableData) + + for msgType in ['ERROR', 'WARNING', 'INFO']: + if msgType in statPack: + tableInfo = [{'header': msgType, 'width': 66, 'just': '<'}, + {'header': 'RecordCount', 'width': 0, 'just': '>', 'style': ','}, + {'header': 'RecordPercent', 'width': 0, 'just': '>', 'style': '.1%'}] + if reportStyle == 'F': + for i in range(10): + tableInfo.append({'header': f'Example: {i+1}', 'width': 50}) + tableData = [] + for itemDict in statPack[msgType]: + rowData = [itemDict['message'], itemDict['recordCount'], itemDict['recordPercent']] + if reportStyle == 'F': + for i in range(10): + if 'topValues' in itemDict and i < len(itemDict['topValues']): + rowData.append(itemDict['topValues'][i]) + else: + rowData.append('') + tableData.append(rowData) + + print() + self.renderTable(tableInfo, tableData) + + if 'ERROR' in statPack or 'WARNING' in statPack: + print('\n** PLEASE REVISIT THE MAPPINGS FOR THIS FILE BEFORE LOADING IT **') + else: + print('\nNO ERRORS OR WARNINGS DETECTED') + + return buf.getvalue() + return '' + + def renderTable(self, tableInfo, tableData): + hdrFormat = '' + rowFormat = '' + headerRow1 = [] + headerRow2 = [] + for columnInfo in tableInfo: + if 'width' not in columnInfo or columnInfo['width'] == 0: + columnInfo['width'] = len(columnInfo['header']) + if 'just' not in columnInfo: + columnInfo['just'] = '<' + hdrFormat += '{:' + columnInfo['just'] + str(columnInfo['width']) + '} ' + rowFormat += '{:' + columnInfo['just'] + str(columnInfo['width']) + (columnInfo['style'] if 'style' in columnInfo else '') + '} ' + headerRow1.append(columnInfo['header']) + headerRow2.append('-' * columnInfo['width']) + + print(hdrFormat.format(*headerRow1)) + print(hdrFormat.format(*headerRow2)) + for row in tableData: + print(rowFormat.format(*row))