From 348ae5e147fd413c283d5ba80283060eef764e8f Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 10 Jan 2022 23:28:14 -0500 Subject: [PATCH 01/13] Remove unused import of fnmatch. --- test/heapwatch/heapWatch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/heapwatch/heapWatch.py b/test/heapwatch/heapWatch.py index d91b780ae4..6119a1ec54 100644 --- a/test/heapwatch/heapWatch.py +++ b/test/heapwatch/heapWatch.py @@ -9,7 +9,6 @@ import argparse import base64 import configparser -import fnmatch import json import logging import os From 2bf4807831cdc6c2af377b242413121cc25463c3 Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 10 Jan 2022 23:32:45 -0500 Subject: [PATCH 02/13] Updated HelloWorld recipe consensus protocol to "future" and added partkeydilution:0. --- .../deployednettemplates/recipes/hello-world/genesis.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json index ac841670ca..b67536670d 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json @@ -1,9 +1,10 @@ { "NetworkName": "", "VersionModifier": "", - "ConsensusProtocol": "", + "ConsensusProtocol": "future", "FirstPartKeyRound": 0, "LastPartKeyRound": 1000300, + "PartKeyDilution": 0, "Wallets": [ { "Name": "Wallet1", From e2027d794d1dd499bde0e4feac81e9be6adb300c Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 11 Jan 2022 18:38:52 -0500 Subject: [PATCH 03/13] Added 1 NPN node to hello-world recipe support pingpong. --- .../recipes/hello-world/genesis.json | 9 +++++-- .../recipes/hello-world/net.json | 25 +++++++++++++++++++ .../recipes/hello-world/topology.json | 4 +++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json index b67536670d..476ce9479e 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json @@ -8,12 +8,17 @@ "Wallets": [ { "Name": "Wallet1", - "Stake": 50, + "Stake": 30, "Online": true }, { "Name": "Wallet2", - "Stake": 50, + "Stake": 30, + "Online": true + }, + { + "Name": "Wallet3", + "Stake": 40, "Online": true } ] diff --git a/test/testdata/deployednettemplates/recipes/hello-world/net.json b/test/testdata/deployednettemplates/recipes/hello-world/net.json index 7d03c46692..eb10536146 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/net.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/net.json @@ -49,6 +49,31 @@ "ConfigJSONOverride": "{ \"DNSBootstrapID\": \".algodev.network\",\"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true }" } ] + }, + { + "Name": "NPN1", + "Nodes": [ + { + "Name": "pingpong node", + "IsRelay": false, + "Wallets": [ + { + "Name": "Wallet3", + "ParticipationOnly": false + } + ], + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"DNSBootstrapID\": \".algodev.network\",\"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true }" + } + ] } ] } diff --git a/test/testdata/deployednettemplates/recipes/hello-world/topology.json b/test/testdata/deployednettemplates/recipes/hello-world/topology.json index ef4f4ac8c6..0894b65bba 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/topology.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/topology.json @@ -7,6 +7,10 @@ { "Name": "R2", "Template": "AWS-US-EAST-1-Small" + }, + { + "Name": "NPN1", + "Template": "AWS-US-EAST-1-Small" } ] } From 96f8a57ac4af85afa48185551d22ffda6973c5e5 Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 11 Jan 2022 20:39:38 -0500 Subject: [PATCH 04/13] Update net.json for hello-world recipe. --- .../recipes/hello-world/genesis.json | 11 ++++-- .../recipes/hello-world/net.json | 37 +++++++++++++++---- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json index 476ce9479e..4ad10407f9 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json @@ -8,17 +8,22 @@ "Wallets": [ { "Name": "Wallet1", - "Stake": 30, + "Stake": 25, "Online": true }, { "Name": "Wallet2", - "Stake": 30, + "Stake": 25, "Online": true }, { "Name": "Wallet3", - "Stake": 40, + "Stake": 25, + "Online": true + }, + { + "Name": "Wallet4", + "Stake": 25, "Online": true } ] diff --git a/test/testdata/deployednettemplates/recipes/hello-world/net.json b/test/testdata/deployednettemplates/recipes/hello-world/net.json index eb10536146..08b885f178 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/net.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/net.json @@ -51,27 +51,48 @@ ] }, { - "Name": "NPN1", + "Name": "N1", + "Group": "", "Nodes": [ { - "Name": "pingpong node", - "IsRelay": false, + "Name": "node1", "Wallets": [ { "Name": "Wallet3", "ParticipationOnly": false } ], - "NetAddress": "{{NetworkPort}}", "APIEndpoint": "{{APIEndpoint}}", "APIToken": "{{APIToken}}", - "EnableTelemetry": true, + "EnableTelemetry": false, "TelemetryURI": "{{TelemetryURI}}", - "EnableMetrics": true, + "EnableMetrics": false, "MetricsURI": "{{MetricsURI}}", "EnableService": false, - "EnableBlockStats": true, - "ConfigJSONOverride": "{ \"DNSBootstrapID\": \".algodev.network\",\"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true }" + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true, \"CadaverSizeTarget\": 0, \"EnableAccountUpdatesStats\": true}" + } + ] + }, + { + "Name": "NPN1", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode1", + "Wallets": [ + { + "Name": "Wallet4", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4, \"CadaverSizeTarget\": 0 }" } ] } From 9acfde67e627983d1252a5b20983c298c805fd90 Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 11 Jan 2022 20:54:34 -0500 Subject: [PATCH 05/13] Update topology.json for hello-world recipe. --- .../deployednettemplates/recipes/hello-world/topology.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/testdata/deployednettemplates/recipes/hello-world/topology.json b/test/testdata/deployednettemplates/recipes/hello-world/topology.json index 0894b65bba..acc7cca9ec 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/topology.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/topology.json @@ -8,6 +8,10 @@ "Name": "R2", "Template": "AWS-US-EAST-1-Small" }, + { + "Name": "N1", + "Template": "AWS-US-EAST-1-Small" + }, { "Name": "NPN1", "Template": "AWS-US-EAST-1-Small" From f612e872999c547f73f87ea5af44c8baa04b4678 Mon Sep 17 00:00:00 2001 From: egieseke Date: Wed, 12 Jan 2022 22:50:23 -0500 Subject: [PATCH 06/13] Catch and ignore exception processing metrics file. --- test/heapwatch/metrics_delta.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 9371591452..36f51a4ba3 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -54,20 +54,23 @@ def test_metric_line_re(): def parse_metrics(fin): out = dict() - for line in fin: - if not line: - continue - line = line.strip() - if not line: - continue - if line[0] == '#': - continue - m = metric_line_re.match(line) - if m: - out[m.group(1)] = num(m.group(2)) - else: - ab = line.split() - out[ab[0]] = num(ab[1]) + try: + for line in fin: + if not line: + continue + line = line.strip() + if not line: + continue + if line[0] == '#': + continue + m = metric_line_re.match(line) + if m: + out[m.group(1)] = num(m.group(2)) + else: + ab = line.split() + out[ab[0]] = num(ab[1]) + except: + print("An exception occurred in parse_metrics") return out # return b-a @@ -396,6 +399,7 @@ def process_files(self, args, nick=None, metrics_files=None): prevbi = None for path in sorted(metrics_files): + print("processing file: ", path) with open(path, 'rt') as fin: cur = parse_metrics(fin) bijsonpath = path.replace('.metrics', '.blockinfo.json') From 9b493061b2afd090dbbe1070fb7ecb1ccb829bdd Mon Sep 17 00:00:00 2001 From: egieseke Date: Thu, 13 Jan 2022 00:10:37 -0500 Subject: [PATCH 07/13] Catch and ignore exception processing metrics file. --- test/heapwatch/metrics_delta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 36f51a4ba3..50881892bf 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -71,6 +71,7 @@ def parse_metrics(fin): out[ab[0]] = num(ab[1]) except: print("An exception occurred in parse_metrics") + pass return out # return b-a From ea5269283ea7759d04ee006e331c613b5a9b92c3 Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 18 Jan 2022 10:40:43 -0500 Subject: [PATCH 08/13] Check for null txp. --- test/heapwatch/metrics_delta.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 50881892bf..6920bf2975 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -197,9 +197,10 @@ def txPool(self): continue # skip the first two while the system could still count as warming up txp = ns.txPool[2:] - mins.append(min(txp)) - maxs.append(max(txp)) - means.append(statistics.mean(txp)) + if not txp is None: + mins.append(min(txp)) + maxs.append(max(txp)) + means.append(statistics.mean(txp)) if not means or not maxs or not mins: return 'txnpool(no stats)' return 'txnpool({:.0f} {:.0f} {:.0f} {:.0f} {:.0f})'.format( From 6a861f1e08397095a28cf452cc6a2392de7ea6ca Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 18 Jan 2022 11:00:47 -0500 Subject: [PATCH 09/13] show exception. --- test/heapwatch/metrics_delta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 6920bf2975..3fbbc863b9 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -70,7 +70,7 @@ def parse_metrics(fin): ab = line.split() out[ab[0]] = num(ab[1]) except: - print("An exception occurred in parse_metrics") + print(f'An exception occurred in parse_metrics {sys.exc_info()[0]}') pass return out From b2cfd76da3b8a5d1750f3b98a315a3afea65e258 Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 18 Jan 2022 13:51:55 -0500 Subject: [PATCH 10/13] show exception. --- test/heapwatch/metrics_delta.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 3fbbc863b9..401e367e22 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -197,10 +197,12 @@ def txPool(self): continue # skip the first two while the system could still count as warming up txp = ns.txPool[2:] - if not txp is None: - mins.append(min(txp)) - maxs.append(max(txp)) - means.append(statistics.mean(txp)) + print(f'txp: {txp}') + if txp is None: + continue + mins.append(min(txp)) + maxs.append(max(txp)) + means.append(statistics.mean(txp)) if not means or not maxs or not mins: return 'txnpool(no stats)' return 'txnpool({:.0f} {:.0f} {:.0f} {:.0f} {:.0f})'.format( From 0b6a15b65f4959c1b583e769a337a7dd04f1e3d9 Mon Sep 17 00:00:00 2001 From: egieseke Date: Tue, 18 Jan 2022 14:02:21 -0500 Subject: [PATCH 11/13] show exception. --- test/heapwatch/metrics_delta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 401e367e22..79cc7b7bc8 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -404,12 +404,12 @@ def process_files(self, args, nick=None, metrics_files=None): for path in sorted(metrics_files): print("processing file: ", path) - with open(path, 'rt') as fin: + with open(path, 'rt', encoding="utf-8") as fin: cur = parse_metrics(fin) bijsonpath = path.replace('.metrics', '.blockinfo.json') bi = None if os.path.exists(bijsonpath): - with open(bijsonpath, 'rt') as fin: + with open(bijsonpath, 'rt', encoding="utf-8") as fin: bi = json.load(fin) curtime = os.path.getmtime(path) self.txPool.append(cur.get('algod_tx_pool_count{}')) From 4f7264e6cd227e8126d91ee991d12109ccd5af80 Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 24 Jan 2022 11:22:57 -0500 Subject: [PATCH 12/13] Removed print statements. --- test/heapwatch/heapWatch.py | 1 + test/heapwatch/metrics_delta.py | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/test/heapwatch/heapWatch.py b/test/heapwatch/heapWatch.py index 6119a1ec54..d91b780ae4 100644 --- a/test/heapwatch/heapWatch.py +++ b/test/heapwatch/heapWatch.py @@ -9,6 +9,7 @@ import argparse import base64 import configparser +import fnmatch import json import logging import os diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 79cc7b7bc8..b6aa2ae8a4 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -70,7 +70,7 @@ def parse_metrics(fin): ab = line.split() out[ab[0]] = num(ab[1]) except: - print(f'An exception occurred in parse_metrics {sys.exc_info()[0]}') + print(f'An exception occurred in parse_metrics: {sys.exc_info()}') pass return out @@ -197,9 +197,6 @@ def txPool(self): continue # skip the first two while the system could still count as warming up txp = ns.txPool[2:] - print(f'txp: {txp}') - if txp is None: - continue mins.append(min(txp)) maxs.append(max(txp)) means.append(statistics.mean(txp)) @@ -403,7 +400,6 @@ def process_files(self, args, nick=None, metrics_files=None): prevbi = None for path in sorted(metrics_files): - print("processing file: ", path) with open(path, 'rt', encoding="utf-8") as fin: cur = parse_metrics(fin) bijsonpath = path.replace('.metrics', '.blockinfo.json') From 04ef3907fb95ba3a0f39220ffc04bec3c674d180 Mon Sep 17 00:00:00 2001 From: egieseke Date: Mon, 24 Jan 2022 12:23:08 -0500 Subject: [PATCH 13/13] Make stake offline for non participating node. --- .../deployednettemplates/recipes/hello-world/genesis.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json index 4ad10407f9..218b694d5f 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json @@ -24,7 +24,7 @@ { "Name": "Wallet4", "Stake": 25, - "Online": true + "Online": false } ] }