From f3000218ee67ac5db979706cb7edaf384b9ae55b Mon Sep 17 00:00:00 2001 From: sukun Date: Sun, 12 Mar 2023 18:20:01 +0530 Subject: [PATCH 1/3] add analysis for protocol versions --- analysis/ipversion_transport.ipynb | 854 +++++++++++++++++++++++++++++ 1 file changed, 854 insertions(+) create mode 100644 analysis/ipversion_transport.ipynb diff --git a/analysis/ipversion_transport.ipynb b/analysis/ipversion_transport.ipynb new file mode 100644 index 0000000..f80ddfe --- /dev/null +++ b/analysis/ipversion_transport.ipynb @@ -0,0 +1,854 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 43, + "id": "20ac4ad3", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import sqlalchemy as sa" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "335afb50", + "metadata": {}, + "outputs": [], + "source": [ + "conn = sa.create_engine(\"postgresql://user:password@localhost:5432/db\")" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "eba0ff4a", + "metadata": {}, + "outputs": [], + "source": [ + "temp_table_q = \"\"\"\n", + " CREATE TEMP TABLE temp_multiaddresses\n", + " AS\n", + " SELECT * from multi_addresses\n", + "\"\"\"\n", + "res = conn.execute(temp_table_q)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "f80286fb", + "metadata": {}, + "outputs": [], + "source": [ + "alter_table_addcols_q = \"\"\"\n", + "ALTER TABLE temp_multiaddresses ADD COLUMN is_ip4_tcp int DEFAULT 0;\n", + "ALTER TABLE temp_multiaddresses ADD COLUMN is_ip4_quic int DEFAULT 0;\n", + "ALTER TABLE temp_multiaddresses ADD COLUMN is_ip6_tcp int DEFAULT 0;\n", + "ALTER TABLE temp_multiaddresses ADD COLUMN is_ip6_quic int DEFAULT 0;\n", + "\"\"\"\n", + "res = conn.execute(alter_table_addcols_q)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "fdb341d9", + "metadata": {}, + "outputs": [], + "source": [ + "update_ip4_tcp_q = \"\"\"\n", + " UPDATE temp_multiaddresses SET\n", + " is_ip4_tcp = CASE\n", + " WHEN family(addr)=4 AND maddr LIKE '%%/tcp/%%' THEN \n", + " 1\n", + " ELSE \n", + " 0\n", + " END;\n", + "\"\"\"\n", + "res = conn.execute(update_ip4_tcp_q)" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "92ce35e8", + "metadata": {}, + "outputs": [], + "source": [ + "update_ip6_tcp_q = \"\"\"\n", + " UPDATE temp_multiaddresses SET\n", + " is_ip6_tcp = CASE\n", + " WHEN family(addr)=6 AND maddr LIKE '%%/tcp/%%' THEN \n", + " 1\n", + " ELSE \n", + " 0\n", + " END;\n", + "\"\"\"\n", + "res = conn.execute(update_ip6_tcp_q)" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "72285405", + "metadata": {}, + "outputs": [], + "source": [ + "update_ip4_quic_q = \"\"\"\n", + "UPDATE temp_multiaddresses SET\n", + " is_ip4_quic = CASE\n", + " WHEN family(addr)=4 AND maddr LIKE '%%/quic%%' THEN\n", + " 1\n", + " ELSE\n", + " 0\n", + " END;\n", + "\"\"\"\n", + "res = conn.execute(update_ip4_quic_q)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "16966d31", + "metadata": {}, + "outputs": [], + "source": [ + "update_ip6_quic_q = \"\"\"\n", + "UPDATE temp_multiaddresses SET\n", + " is_ip6_quic = CASE\n", + " WHEN family(addr)=6 AND maddr LIKE '%%/quic%%' THEN\n", + " 1\n", + " ELSE\n", + " 0\n", + " END;\n", + "\"\"\"\n", + "res = conn.execute(update_ip6_quic_q)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "d9f9d624", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
outcomeprotocol_filterslocal_is_ip4_tcplocal_is_ip6_tcplocal_is_ip4_quiclocal_is_ip6_quicremote_is_ip4_tcpremote_is_ip6_tcpremote_is_ip4_quicremote_is_ip6_quiccount
0NO_STREAM[]000100102
1NO_STREAM[]000110003
2NO_STREAM[]000110105
3NO_STREAM[]0010001011
4NO_STREAM[]001000112
....................................
1463SUCCESS[41, 460]1111101034836
1464SUCCESS[41, 460]1111101110
1465SUCCESS[41, 460]111111002119
1466SUCCESS[41, 460]111111102011
1467SUCCESS[41, 460]111111112
\n", + "

1468 rows × 11 columns

\n", + "
" + ], + "text/plain": [ + " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", + "0 NO_STREAM [] 0 0 \n", + "1 NO_STREAM [] 0 0 \n", + "2 NO_STREAM [] 0 0 \n", + "3 NO_STREAM [] 0 0 \n", + "4 NO_STREAM [] 0 0 \n", + "... ... ... ... ... \n", + "1463 SUCCESS [41, 460] 1 1 \n", + "1464 SUCCESS [41, 460] 1 1 \n", + "1465 SUCCESS [41, 460] 1 1 \n", + "1466 SUCCESS [41, 460] 1 1 \n", + "1467 SUCCESS [41, 460] 1 1 \n", + "\n", + " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", + "0 0 1 0 \n", + "1 0 1 1 \n", + "2 0 1 1 \n", + "3 1 0 0 \n", + "4 1 0 0 \n", + "... ... ... ... \n", + "1463 1 1 1 \n", + "1464 1 1 1 \n", + "1465 1 1 1 \n", + "1466 1 1 1 \n", + "1467 1 1 1 \n", + "\n", + " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \n", + "0 0 1 0 2 \n", + "1 0 0 0 3 \n", + "2 0 1 0 5 \n", + "3 0 1 0 11 \n", + "4 0 1 1 2 \n", + "... ... ... ... ... \n", + "1463 0 1 0 34836 \n", + "1464 0 1 1 10 \n", + "1465 1 0 0 2119 \n", + "1466 1 1 0 2011 \n", + "1467 1 1 1 2 \n", + "\n", + "[1468 rows x 11 columns]" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "final_results_q = \"\"\"\n", + "SELECT \n", + " hpr.outcome, \n", + " hpr.protocol_filters, \n", + " hpr.local_is_ip4_tcp,\n", + " hpr.local_is_ip6_tcp,\n", + " hpr.local_is_ip4_quic,\n", + " hpr.local_is_ip6_quic,\n", + " hpa.is_ip4_tcp as remote_is_ip4_tcp,\n", + " hpa.is_ip6_tcp as remote_is_ip6_tcp,\n", + " hpa.is_ip4_quic as remote_is_ip4_quic,\n", + " hpa.is_ip6_quic as remote_is_ip6_quic,\n", + " COUNT(DISTINCT hpr.id) as count\n", + " FROM (\n", + " (SELECT\n", + " hpr.id, \n", + " hpr.outcome,\n", + " hpr.protocol_filters,\n", + " max(ma.is_ip4_tcp) as local_is_ip4_tcp, \n", + " max(ma.is_ip6_tcp) as local_is_ip6_tcp, \n", + " max(ma.is_ip4_quic) as local_is_ip4_quic,\n", + " max(ma.is_ip6_quic) as local_is_ip6_quic\n", + " FROM\n", + " hole_punch_results hpr\n", + " INNER JOIN multi_addresses_sets mas ON mas.id=hpr.listen_multi_addresses_set_id\n", + " CROSS JOIN unnest(mas.multi_addresses_ids) lmids(mid)\n", + " INNER JOIN temp_multiaddresses ma ON ma.id=lmids.mid\n", + " WHERE \n", + " NOT ma.is_relay \n", + " AND ma.is_public\n", + " AND NOT EXISTS (SELECT FROM port_mappings pm WHERE pm.hole_punch_result_id = hpr.id)\n", + " GROUP BY hpr.id, hpr.outcome, hpr.protocol_filters\n", + " ) hpr\n", + " INNER JOIN (\n", + " SELECT \n", + " hpa.hole_punch_result_id, \n", + " max(ma.is_ip4_tcp) as is_ip4_tcp, \n", + " max(ma.is_ip6_tcp) as is_ip6_tcp, \n", + " max(ma.is_ip4_quic) as is_ip4_quic,\n", + " max(ma.is_ip6_quic) as is_ip6_quic\n", + " FROM hole_punch_attempt hpa\n", + " INNER JOIN hole_punch_attempt_x_multi_addresses hpaxma on hpa.id = hpaxma.hole_punch_attempt\n", + " INNER JOIN temp_multiaddresses ma on hpaxma.multi_address_id = ma.id\n", + " WHERE ma.is_public AND NOT ma.is_relay\n", + " GROUP BY hpa.hole_punch_result_id\n", + " ) hpa ON hpa.hole_punch_result_id=hpr.id\n", + " )\n", + " GROUP BY \n", + " outcome, \n", + " protocol_filters, \n", + " local_is_ip4_tcp,\n", + " local_is_ip6_tcp,\n", + " local_is_ip4_quic,\n", + " local_is_ip6_quic,\n", + " remote_is_ip4_tcp,\n", + " remote_is_ip6_tcp,\n", + " remote_is_ip4_quic,\n", + " remote_is_ip6_quic\n", + "\"\"\"\n", + "df = pd.read_sql(final_results_q, con=conn)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "6345362c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
outcomeprotocol_filterslocal_is_ip4_tcplocal_is_ip6_tcplocal_is_ip4_quiclocal_is_ip6_quicremote_is_ip4_tcpremote_is_ip6_tcpremote_is_ip4_quicremote_is_ip6_quiccountpf
0NO_STREAM[]000100102No Filter
1NO_STREAM[]000110003No Filter
2NO_STREAM[]000110105No Filter
3NO_STREAM[]0010001011No Filter
4NO_STREAM[]001000112No Filter
.......................................
1463SUCCESS[41, 460]1111101034836ip:6|transport:quic
1464SUCCESS[41, 460]1111101110ip:6|transport:quic
1465SUCCESS[41, 460]111111002119ip:6|transport:quic
1466SUCCESS[41, 460]111111102011ip:6|transport:quic
1467SUCCESS[41, 460]111111112ip:6|transport:quic
\n", + "

1468 rows × 12 columns

\n", + "
" + ], + "text/plain": [ + " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", + "0 NO_STREAM [] 0 0 \n", + "1 NO_STREAM [] 0 0 \n", + "2 NO_STREAM [] 0 0 \n", + "3 NO_STREAM [] 0 0 \n", + "4 NO_STREAM [] 0 0 \n", + "... ... ... ... ... \n", + "1463 SUCCESS [41, 460] 1 1 \n", + "1464 SUCCESS [41, 460] 1 1 \n", + "1465 SUCCESS [41, 460] 1 1 \n", + "1466 SUCCESS [41, 460] 1 1 \n", + "1467 SUCCESS [41, 460] 1 1 \n", + "\n", + " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", + "0 0 1 0 \n", + "1 0 1 1 \n", + "2 0 1 1 \n", + "3 1 0 0 \n", + "4 1 0 0 \n", + "... ... ... ... \n", + "1463 1 1 1 \n", + "1464 1 1 1 \n", + "1465 1 1 1 \n", + "1466 1 1 1 \n", + "1467 1 1 1 \n", + "\n", + " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \\\n", + "0 0 1 0 2 \n", + "1 0 0 0 3 \n", + "2 0 1 0 5 \n", + "3 0 1 0 11 \n", + "4 0 1 1 2 \n", + "... ... ... ... ... \n", + "1463 0 1 0 34836 \n", + "1464 0 1 1 10 \n", + "1465 1 0 0 2119 \n", + "1466 1 1 0 2011 \n", + "1467 1 1 1 2 \n", + "\n", + " pf \n", + "0 No Filter \n", + "1 No Filter \n", + "2 No Filter \n", + "3 No Filter \n", + "4 No Filter \n", + "... ... \n", + "1463 ip:6|transport:quic \n", + "1464 ip:6|transport:quic \n", + "1465 ip:6|transport:quic \n", + "1466 ip:6|transport:quic \n", + "1467 ip:6|transport:quic \n", + "\n", + "[1468 rows x 12 columns]" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def pf_to_string(x):\n", + " if not x['protocol_filters']:\n", + " return \"No Filter\"\n", + " else:\n", + " ip_filter = 4 if x['protocol_filters'][0] == 4 else 6\n", + " transport_filter = \"tcp\" if x['protocol_filters'][1] == 6 else \"quic\"\n", + " return f\"ip:{ip_filter}|transport:{transport_filter}\"\n", + "dff['pf'] = dff.apply(lambda x: pf_to_string(x), axis=1)\n", + "dff" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "f5fe889c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "pf outcome \n", + "ip:6|transport:tcp CONNECTION_REVERSED 390\n", + " FAILED 21722\n", + " NO_STREAM 346\n", + " SUCCESS 9624\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dff.query('pf == \"ip:6|transport:tcp\" & local_is_ip6_tcp==1 & remote_is_ip6_tcp==1').groupby(['pf', 'outcome'])['count'].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "036f7904", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "pf outcome \n", + "ip:4|transport:tcp CONNECTION_REVERSED 2046\n", + " FAILED 108058\n", + " NO_STREAM 2647\n", + " SUCCESS 313588\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dff.query('pf == \"ip:4|transport:tcp\" & local_is_ip4_tcp==1 & remote_is_ip4_tcp==1').groupby(['pf', 'outcome'])['count'].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "ab4b39c4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "pf outcome \n", + "ip:6|transport:quic CONNECTION_REVERSED 415\n", + " FAILED 30736\n", + " NO_STREAM 429\n", + " SUCCESS 9385\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dff.query('pf == \"ip:6|transport:quic\" & local_is_ip6_quic==1 & remote_is_ip6_quic==1').groupby(['pf', 'outcome'])['count'].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "5d0e2488", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "pf outcome \n", + "ip:4|transport:quic CONNECTION_REVERSED 2718\n", + " FAILED 116924\n", + " NO_STREAM 2705\n", + " SUCCESS 259798\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dff.query('pf == \"ip:4|transport:quic\" & local_is_ip4_quic==1 & remote_is_ip4_quic==1').groupby(['pf', 'outcome'])['count'].sum()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 3feff7674ab48ff9a8d2c5ab310a69c0d3ecabbe Mon Sep 17 00:00:00 2001 From: sukun Date: Mon, 13 Mar 2023 21:54:27 +0530 Subject: [PATCH 2/3] update analysis to take count of addresses --- analysis/ipversion_transport.ipynb | 498 ++++++++++++++++------------- 1 file changed, 276 insertions(+), 222 deletions(-) diff --git a/analysis/ipversion_transport.ipynb b/analysis/ipversion_transport.ipynb index f80ddfe..ce5cdbd 100644 --- a/analysis/ipversion_transport.ipynb +++ b/analysis/ipversion_transport.ipynb @@ -2,10 +2,19 @@ "cells": [ { "cell_type": "code", - "execution_count": 43, + "execution_count": 1, "id": "20ac4ad3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/sukun/notebooks/venv/lib/python3.9/site-packages/pandas/compat/__init__.py:124: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n", + " warnings.warn(msg)\n" + ] + } + ], "source": [ "import pandas as pd\n", "import sqlalchemy as sa" @@ -13,20 +22,29 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 2, "id": "335afb50", "metadata": {}, "outputs": [], "source": [ - "conn = sa.create_engine(\"postgresql://user:password@localhost:5432/db\")" + "conn = sa.create_engine(\"postgresql://user:pwd@localhost:5432/db\")" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 3, "id": "eba0ff4a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/kk/51zcty2n1093n0kt7k9d17240000gn/T/ipykernel_92001/910345355.py:6: RemovedIn20Warning: Deprecated API features detected! These feature(s) are not compatible with SQLAlchemy 2.0. To prevent incompatible upgrades prior to updating applications, ensure requirements files are pinned to \"sqlalchemy<2.0\". Set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings. Set environment variable SQLALCHEMY_SILENCE_UBER_WARNING=1 to silence this message. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9)\n", + " res = conn.execute(temp_table_q)\n" + ] + } + ], "source": [ "temp_table_q = \"\"\"\n", " CREATE TEMP TABLE temp_multiaddresses\n", @@ -38,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 4, "id": "f80286fb", "metadata": {}, "outputs": [], @@ -54,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 5, "id": "fdb341d9", "metadata": {}, "outputs": [], @@ -73,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 6, "id": "92ce35e8", "metadata": {}, "outputs": [], @@ -92,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 7, "id": "72285405", "metadata": {}, "outputs": [], @@ -111,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 8, "id": "16966d31", "metadata": {}, "outputs": [], @@ -130,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 9, "id": "d9f9d624", "metadata": {}, "outputs": [ @@ -179,9 +197,9 @@ " 1\n", " 0\n", " 0\n", - " 1\n", - " 0\n", " 2\n", + " 0\n", + " 1\n", " \n", " \n", " 1\n", @@ -191,11 +209,11 @@ " 0\n", " 0\n", " 1\n", - " 1\n", " 0\n", " 0\n", + " 6\n", " 0\n", - " 3\n", + " 1\n", " \n", " \n", " 2\n", @@ -207,9 +225,9 @@ " 1\n", " 1\n", " 0\n", - " 1\n", " 0\n", - " 5\n", + " 0\n", + " 1\n", " \n", " \n", " 3\n", @@ -217,13 +235,13 @@ " []\n", " 0\n", " 0\n", - " 1\n", - " 0\n", " 0\n", + " 1\n", + " 1\n", " 0\n", " 1\n", " 0\n", - " 11\n", + " 2\n", " \n", " \n", " 4\n", @@ -231,13 +249,13 @@ " []\n", " 0\n", " 0\n", + " 0\n", " 1\n", + " 2\n", " 0\n", " 0\n", " 0\n", " 1\n", - " 1\n", - " 2\n", " \n", " \n", " ...\n", @@ -254,124 +272,124 @@ " ...\n", " \n", " \n", - " 1463\n", + " 20727\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", + " 0\n", " 1\n", " 0\n", + " 2\n", " 1\n", + " 2\n", " 0\n", - " 34836\n", + " 1\n", " \n", " \n", - " 1464\n", + " 20728\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", " 0\n", " 1\n", + " 0\n", + " 4\n", + " 0\n", + " 0\n", + " 0\n", " 1\n", - " 10\n", " \n", " \n", - " 1465\n", + " 20729\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", + " 0\n", + " 2\n", + " 0\n", " 1\n", " 0\n", + " 1\n", " 0\n", - " 2119\n", + " 1\n", " \n", " \n", - " 1466\n", + " 20730\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", + " 0\n", + " 2\n", " 0\n", - " 2011\n", + " 2\n", + " 0\n", + " 0\n", + " 0\n", + " 1\n", " \n", " \n", - " 1467\n", + " 20731\n", " SUCCESS\n", " [41, 460]\n", + " 3\n", " 1\n", + " 2\n", + " 0\n", " 1\n", + " 0\n", " 1\n", + " 0\n", " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 2\n", " \n", " \n", "\n", - "

1468 rows × 11 columns

\n", + "

20732 rows × 11 columns

\n", "" ], "text/plain": [ - " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", - "0 NO_STREAM [] 0 0 \n", - "1 NO_STREAM [] 0 0 \n", - "2 NO_STREAM [] 0 0 \n", - "3 NO_STREAM [] 0 0 \n", - "4 NO_STREAM [] 0 0 \n", - "... ... ... ... ... \n", - "1463 SUCCESS [41, 460] 1 1 \n", - "1464 SUCCESS [41, 460] 1 1 \n", - "1465 SUCCESS [41, 460] 1 1 \n", - "1466 SUCCESS [41, 460] 1 1 \n", - "1467 SUCCESS [41, 460] 1 1 \n", + " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", + "0 NO_STREAM [] 0 0 \n", + "1 NO_STREAM [] 0 0 \n", + "2 NO_STREAM [] 0 0 \n", + "3 NO_STREAM [] 0 0 \n", + "4 NO_STREAM [] 0 0 \n", + "... ... ... ... ... \n", + "20727 SUCCESS [41, 460] 3 0 \n", + "20728 SUCCESS [41, 460] 3 0 \n", + "20729 SUCCESS [41, 460] 3 0 \n", + "20730 SUCCESS [41, 460] 3 0 \n", + "20731 SUCCESS [41, 460] 3 1 \n", "\n", - " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", - "0 0 1 0 \n", - "1 0 1 1 \n", - "2 0 1 1 \n", - "3 1 0 0 \n", - "4 1 0 0 \n", - "... ... ... ... \n", - "1463 1 1 1 \n", - "1464 1 1 1 \n", - "1465 1 1 1 \n", - "1466 1 1 1 \n", - "1467 1 1 1 \n", + " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", + "0 0 1 0 \n", + "1 0 1 0 \n", + "2 0 1 1 \n", + "3 0 1 1 \n", + "4 0 1 2 \n", + "... ... ... ... \n", + "20727 1 0 2 \n", + "20728 1 0 4 \n", + "20729 2 0 1 \n", + "20730 2 0 2 \n", + "20731 2 0 1 \n", "\n", - " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \n", - "0 0 1 0 2 \n", - "1 0 0 0 3 \n", - "2 0 1 0 5 \n", - "3 0 1 0 11 \n", - "4 0 1 1 2 \n", - "... ... ... ... ... \n", - "1463 0 1 0 34836 \n", - "1464 0 1 1 10 \n", - "1465 1 0 0 2119 \n", - "1466 1 1 0 2011 \n", - "1467 1 1 1 2 \n", + " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \n", + "0 0 2 0 1 \n", + "1 0 6 0 1 \n", + "2 0 0 0 1 \n", + "3 0 1 0 2 \n", + "4 0 0 0 1 \n", + "... ... ... ... ... \n", + "20727 1 2 0 1 \n", + "20728 0 0 0 1 \n", + "20729 0 1 0 1 \n", + "20730 0 0 0 1 \n", + "20731 0 1 0 1 \n", "\n", - "[1468 rows x 11 columns]" + "[20732 rows x 11 columns]" ] }, - "execution_count": 55, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -395,10 +413,10 @@ " hpr.id, \n", " hpr.outcome,\n", " hpr.protocol_filters,\n", - " max(ma.is_ip4_tcp) as local_is_ip4_tcp, \n", - " max(ma.is_ip6_tcp) as local_is_ip6_tcp, \n", - " max(ma.is_ip4_quic) as local_is_ip4_quic,\n", - " max(ma.is_ip6_quic) as local_is_ip6_quic\n", + " sum(ma.is_ip4_tcp) as local_is_ip4_tcp, \n", + " sum(ma.is_ip6_tcp) as local_is_ip6_tcp, \n", + " sum(ma.is_ip4_quic) as local_is_ip4_quic,\n", + " sum(ma.is_ip6_quic) as local_is_ip6_quic\n", " FROM\n", " hole_punch_results hpr\n", " INNER JOIN multi_addresses_sets mas ON mas.id=hpr.listen_multi_addresses_set_id\n", @@ -413,10 +431,10 @@ " INNER JOIN (\n", " SELECT \n", " hpa.hole_punch_result_id, \n", - " max(ma.is_ip4_tcp) as is_ip4_tcp, \n", - " max(ma.is_ip6_tcp) as is_ip6_tcp, \n", - " max(ma.is_ip4_quic) as is_ip4_quic,\n", - " max(ma.is_ip6_quic) as is_ip6_quic\n", + " sum(ma.is_ip4_tcp) as is_ip4_tcp, \n", + " sum(ma.is_ip6_tcp) as is_ip6_tcp, \n", + " sum(ma.is_ip4_quic) as is_ip4_quic,\n", + " sum(ma.is_ip6_quic) as is_ip6_quic\n", " FROM hole_punch_attempt hpa\n", " INNER JOIN hole_punch_attempt_x_multi_addresses hpaxma on hpa.id = hpaxma.hole_punch_attempt\n", " INNER JOIN temp_multiaddresses ma on hpaxma.multi_address_id = ma.id\n", @@ -442,7 +460,17 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 11, + "id": "fd2f445d", + "metadata": {}, + "outputs": [], + "source": [ + "dff = df" + ] + }, + { + "cell_type": "code", + "execution_count": 12, "id": "6345362c", "metadata": {}, "outputs": [ @@ -492,9 +520,9 @@ " 1\n", " 0\n", " 0\n", - " 1\n", - " 0\n", " 2\n", + " 0\n", + " 1\n", " No Filter\n", " \n", " \n", @@ -505,11 +533,11 @@ " 0\n", " 0\n", " 1\n", - " 1\n", " 0\n", " 0\n", + " 6\n", " 0\n", - " 3\n", + " 1\n", " No Filter\n", " \n", " \n", @@ -522,9 +550,9 @@ " 1\n", " 1\n", " 0\n", - " 1\n", " 0\n", - " 5\n", + " 0\n", + " 1\n", " No Filter\n", " \n", " \n", @@ -533,13 +561,13 @@ " []\n", " 0\n", " 0\n", - " 1\n", - " 0\n", " 0\n", + " 1\n", + " 1\n", " 0\n", " 1\n", " 0\n", - " 11\n", + " 2\n", " No Filter\n", " \n", " \n", @@ -548,13 +576,13 @@ " []\n", " 0\n", " 0\n", + " 0\n", " 1\n", + " 2\n", " 0\n", " 0\n", " 0\n", " 1\n", - " 1\n", - " 2\n", " No Filter\n", " \n", " \n", @@ -573,142 +601,142 @@ " ...\n", " \n", " \n", - " 1463\n", + " 20727\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", + " 0\n", " 1\n", " 0\n", + " 2\n", " 1\n", + " 2\n", " 0\n", - " 34836\n", + " 1\n", " ip:6|transport:quic\n", " \n", " \n", - " 1464\n", + " 20728\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", " 0\n", " 1\n", + " 0\n", + " 4\n", + " 0\n", + " 0\n", + " 0\n", " 1\n", - " 10\n", " ip:6|transport:quic\n", " \n", " \n", - " 1465\n", + " 20729\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", + " 0\n", + " 2\n", + " 0\n", " 1\n", " 0\n", + " 1\n", " 0\n", - " 2119\n", + " 1\n", " ip:6|transport:quic\n", " \n", " \n", - " 1466\n", + " 20730\n", " SUCCESS\n", " [41, 460]\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", + " 3\n", + " 0\n", + " 2\n", " 0\n", - " 2011\n", + " 2\n", + " 0\n", + " 0\n", + " 0\n", + " 1\n", " ip:6|transport:quic\n", " \n", " \n", - " 1467\n", + " 20731\n", " SUCCESS\n", " [41, 460]\n", + " 3\n", " 1\n", + " 2\n", + " 0\n", " 1\n", + " 0\n", " 1\n", + " 0\n", " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 1\n", - " 2\n", " ip:6|transport:quic\n", " \n", " \n", "\n", - "

1468 rows × 12 columns

\n", + "

20732 rows × 12 columns

\n", "" ], "text/plain": [ - " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", - "0 NO_STREAM [] 0 0 \n", - "1 NO_STREAM [] 0 0 \n", - "2 NO_STREAM [] 0 0 \n", - "3 NO_STREAM [] 0 0 \n", - "4 NO_STREAM [] 0 0 \n", - "... ... ... ... ... \n", - "1463 SUCCESS [41, 460] 1 1 \n", - "1464 SUCCESS [41, 460] 1 1 \n", - "1465 SUCCESS [41, 460] 1 1 \n", - "1466 SUCCESS [41, 460] 1 1 \n", - "1467 SUCCESS [41, 460] 1 1 \n", + " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", + "0 NO_STREAM [] 0 0 \n", + "1 NO_STREAM [] 0 0 \n", + "2 NO_STREAM [] 0 0 \n", + "3 NO_STREAM [] 0 0 \n", + "4 NO_STREAM [] 0 0 \n", + "... ... ... ... ... \n", + "20727 SUCCESS [41, 460] 3 0 \n", + "20728 SUCCESS [41, 460] 3 0 \n", + "20729 SUCCESS [41, 460] 3 0 \n", + "20730 SUCCESS [41, 460] 3 0 \n", + "20731 SUCCESS [41, 460] 3 1 \n", "\n", - " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", - "0 0 1 0 \n", - "1 0 1 1 \n", - "2 0 1 1 \n", - "3 1 0 0 \n", - "4 1 0 0 \n", - "... ... ... ... \n", - "1463 1 1 1 \n", - "1464 1 1 1 \n", - "1465 1 1 1 \n", - "1466 1 1 1 \n", - "1467 1 1 1 \n", + " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", + "0 0 1 0 \n", + "1 0 1 0 \n", + "2 0 1 1 \n", + "3 0 1 1 \n", + "4 0 1 2 \n", + "... ... ... ... \n", + "20727 1 0 2 \n", + "20728 1 0 4 \n", + "20729 2 0 1 \n", + "20730 2 0 2 \n", + "20731 2 0 1 \n", "\n", - " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \\\n", - "0 0 1 0 2 \n", - "1 0 0 0 3 \n", - "2 0 1 0 5 \n", - "3 0 1 0 11 \n", - "4 0 1 1 2 \n", - "... ... ... ... ... \n", - "1463 0 1 0 34836 \n", - "1464 0 1 1 10 \n", - "1465 1 0 0 2119 \n", - "1466 1 1 0 2011 \n", - "1467 1 1 1 2 \n", + " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \\\n", + "0 0 2 0 1 \n", + "1 0 6 0 1 \n", + "2 0 0 0 1 \n", + "3 0 1 0 2 \n", + "4 0 0 0 1 \n", + "... ... ... ... ... \n", + "20727 1 2 0 1 \n", + "20728 0 0 0 1 \n", + "20729 0 1 0 1 \n", + "20730 0 0 0 1 \n", + "20731 0 1 0 1 \n", "\n", - " pf \n", - "0 No Filter \n", - "1 No Filter \n", - "2 No Filter \n", - "3 No Filter \n", - "4 No Filter \n", - "... ... \n", - "1463 ip:6|transport:quic \n", - "1464 ip:6|transport:quic \n", - "1465 ip:6|transport:quic \n", - "1466 ip:6|transport:quic \n", - "1467 ip:6|transport:quic \n", + " pf \n", + "0 No Filter \n", + "1 No Filter \n", + "2 No Filter \n", + "3 No Filter \n", + "4 No Filter \n", + "... ... \n", + "20727 ip:6|transport:quic \n", + "20728 ip:6|transport:quic \n", + "20729 ip:6|transport:quic \n", + "20730 ip:6|transport:quic \n", + "20731 ip:6|transport:quic \n", "\n", - "[1468 rows x 12 columns]" + "[20732 rows x 12 columns]" ] }, - "execution_count": 58, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -727,7 +755,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 17, "id": "f5fe889c", "metadata": {}, "outputs": [ @@ -735,14 +763,14 @@ "data": { "text/plain": [ "pf outcome \n", - "ip:6|transport:tcp CONNECTION_REVERSED 390\n", - " FAILED 21722\n", - " NO_STREAM 346\n", - " SUCCESS 9624\n", + "ip:6|transport:tcp CONNECTION_REVERSED 324\n", + " FAILED 216\n", + " NO_STREAM 200\n", + " SUCCESS 7739\n", "Name: count, dtype: int64" ] }, - "execution_count": 59, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -753,7 +781,33 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 26, + "id": "012385d4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "pf outcome \n", + "ip:6|transport:tcp CONNECTION_REVERSED 64\n", + " FAILED 21194\n", + " NO_STREAM 142\n", + " SUCCESS 1744\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dff.query('pf == \"ip:6|transport:tcp\" & local_is_ip6_tcp==1 & remote_is_ip6_tcp>1').groupby(['pf', 'outcome'])['count'].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, "id": "036f7904", "metadata": {}, "outputs": [ @@ -761,14 +815,14 @@ "data": { "text/plain": [ "pf outcome \n", - "ip:4|transport:tcp CONNECTION_REVERSED 2046\n", - " FAILED 108058\n", - " NO_STREAM 2647\n", - " SUCCESS 313588\n", + "ip:4|transport:tcp CONNECTION_REVERSED 1445\n", + " FAILED 792\n", + " NO_STREAM 857\n", + " SUCCESS 266724\n", "Name: count, dtype: int64" ] }, - "execution_count": 60, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -779,7 +833,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 25, "id": "ab4b39c4", "metadata": {}, "outputs": [ @@ -787,20 +841,20 @@ "data": { "text/plain": [ "pf outcome \n", - "ip:6|transport:quic CONNECTION_REVERSED 415\n", - " FAILED 30736\n", - " NO_STREAM 429\n", - " SUCCESS 9385\n", + "ip:6|transport:quic CONNECTION_REVERSED 66\n", + " FAILED 30233\n", + " NO_STREAM 192\n", + " SUCCESS 2102\n", "Name: count, dtype: int64" ] }, - "execution_count": 61, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dff.query('pf == \"ip:6|transport:quic\" & local_is_ip6_quic==1 & remote_is_ip6_quic==1').groupby(['pf', 'outcome'])['count'].sum()" + "dff.query('pf == \"ip:6|transport:quic\" & local_is_ip6_quic==1 & remote_is_ip6_quic>1').groupby(['pf', 'outcome'])['count'].sum()" ] }, { From 32d058fb9817b0d10fd2816780c8f02f87808a3c Mon Sep 17 00:00:00 2001 From: sukun Date: Mon, 20 Mar 2023 15:13:19 +0530 Subject: [PATCH 3/3] fix bug with address counting --- analysis/ipversion_transport.ipynb | 808 +++++++---------------------- 1 file changed, 186 insertions(+), 622 deletions(-) diff --git a/analysis/ipversion_transport.ipynb b/analysis/ipversion_transport.ipynb index ce5cdbd..2a09d82 100644 --- a/analysis/ipversion_transport.ipynb +++ b/analysis/ipversion_transport.ipynb @@ -2,19 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 258, "id": "20ac4ad3", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/sukun/notebooks/venv/lib/python3.9/site-packages/pandas/compat/__init__.py:124: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n", - " warnings.warn(msg)\n" - ] - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "import sqlalchemy as sa" @@ -22,7 +13,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 259, "id": "335afb50", "metadata": {}, "outputs": [], @@ -32,19 +23,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 260, "id": "eba0ff4a", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/kk/51zcty2n1093n0kt7k9d17240000gn/T/ipykernel_92001/910345355.py:6: RemovedIn20Warning: Deprecated API features detected! These feature(s) are not compatible with SQLAlchemy 2.0. To prevent incompatible upgrades prior to updating applications, ensure requirements files are pinned to \"sqlalchemy<2.0\". Set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings. Set environment variable SQLALCHEMY_SILENCE_UBER_WARNING=1 to silence this message. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9)\n", - " res = conn.execute(temp_table_q)\n" - ] - } - ], + "outputs": [], "source": [ "temp_table_q = \"\"\"\n", " CREATE TEMP TABLE temp_multiaddresses\n", @@ -54,9 +36,21 @@ "res = conn.execute(temp_table_q)\n" ] }, + { + "cell_type": "markdown", + "id": "43e79dd2", + "metadata": {}, + "source": [ + "Insert fields to indicate the type of given multiaddress: \n", + "- ip4_tcp\n", + "- ip4_quic\n", + "- ip6_tcp\n", + "- ip6_quic" + ] + }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 261, "id": "f80286fb", "metadata": {}, "outputs": [], @@ -72,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 262, "id": "fdb341d9", "metadata": {}, "outputs": [], @@ -91,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 263, "id": "92ce35e8", "metadata": {}, "outputs": [], @@ -110,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 264, "id": "72285405", "metadata": {}, "outputs": [], @@ -129,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 265, "id": "16966d31", "metadata": {}, "outputs": [], @@ -146,277 +140,49 @@ "res = conn.execute(update_ip6_quic_q)" ] }, + { + "cell_type": "markdown", + "id": "bb87f705", + "metadata": {}, + "source": [ + "We are only interested in holepunch results where both the sides had an address relevant for a given protocol filter\n", + "\n", + "if we applied a filter of \\[ip6, quic\\] we'd like to only look at results where both the sides had at least one ip6_quic address.\n", + "\n", + "In the query: \n", + "The first subquery calculates the number of address types(count ip6_quic etc) for the local side\n", + "The second subquery calculates the number of address types for the remote side" + ] + }, { "cell_type": "code", - "execution_count": 9, - "id": "d9f9d624", + "execution_count": 288, + "id": "b2018df8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
outcomeprotocol_filterslocal_is_ip4_tcplocal_is_ip6_tcplocal_is_ip4_quiclocal_is_ip6_quicremote_is_ip4_tcpremote_is_ip6_tcpremote_is_ip4_quicremote_is_ip6_quiccount
0NO_STREAM[]000100201
1NO_STREAM[]000100601
2NO_STREAM[]000110001
3NO_STREAM[]000110102
4NO_STREAM[]000120001
....................................
20727SUCCESS[41, 460]301021201
20728SUCCESS[41, 460]301040001
20729SUCCESS[41, 460]302010101
20730SUCCESS[41, 460]302020001
20731SUCCESS[41, 460]312010101
\n", - "

20732 rows × 11 columns

\n", - "
" - ], - "text/plain": [ - " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", - "0 NO_STREAM [] 0 0 \n", - "1 NO_STREAM [] 0 0 \n", - "2 NO_STREAM [] 0 0 \n", - "3 NO_STREAM [] 0 0 \n", - "4 NO_STREAM [] 0 0 \n", - "... ... ... ... ... \n", - "20727 SUCCESS [41, 460] 3 0 \n", - "20728 SUCCESS [41, 460] 3 0 \n", - "20729 SUCCESS [41, 460] 3 0 \n", - "20730 SUCCESS [41, 460] 3 0 \n", - "20731 SUCCESS [41, 460] 3 1 \n", - "\n", - " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", - "0 0 1 0 \n", - "1 0 1 0 \n", - "2 0 1 1 \n", - "3 0 1 1 \n", - "4 0 1 2 \n", - "... ... ... ... \n", - "20727 1 0 2 \n", - "20728 1 0 4 \n", - "20729 2 0 1 \n", - "20730 2 0 2 \n", - "20731 2 0 1 \n", - "\n", - " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \n", - "0 0 2 0 1 \n", - "1 0 6 0 1 \n", - "2 0 0 0 1 \n", - "3 0 1 0 2 \n", - "4 0 0 0 1 \n", - "... ... ... ... ... \n", - "20727 1 2 0 1 \n", - "20728 0 0 0 1 \n", - "20729 0 1 0 1 \n", - "20730 0 0 0 1 \n", - "20731 0 1 0 1 \n", - "\n", - "[20732 rows x 11 columns]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "final_results_q = \"\"\"\n", "SELECT \n", " hpr.outcome, \n", " hpr.protocol_filters, \n", - " hpr.local_is_ip4_tcp,\n", - " hpr.local_is_ip6_tcp,\n", - " hpr.local_is_ip4_quic,\n", - " hpr.local_is_ip6_quic,\n", - " hpa.is_ip4_tcp as remote_is_ip4_tcp,\n", - " hpa.is_ip6_tcp as remote_is_ip6_tcp,\n", - " hpa.is_ip4_quic as remote_is_ip4_quic,\n", - " hpa.is_ip6_quic as remote_is_ip6_quic,\n", + " hpr.local_ip4_tcp_cnt,\n", + " hpr.local_ip6_tcp_cnt,\n", + " hpr.local_ip4_quic_cnt,\n", + " hpr.local_ip6_quic_cnt,\n", + " remote_addrs.ip4_tcp_cnt as remote_ip4_tcp_cnt,\n", + " remote_addrs.ip6_tcp_cnt as remote_ip6_tcp_cnt,\n", + " remote_addrs.ip4_quic_cnt as remote_ip4_quic_cnt,\n", + " remote_addrs.ip6_quic_cnt as remote_ip6_quic_cnt,\n", " COUNT(DISTINCT hpr.id) as count\n", " FROM (\n", " (SELECT\n", " hpr.id, \n", " hpr.outcome,\n", " hpr.protocol_filters,\n", - " sum(ma.is_ip4_tcp) as local_is_ip4_tcp, \n", - " sum(ma.is_ip6_tcp) as local_is_ip6_tcp, \n", - " sum(ma.is_ip4_quic) as local_is_ip4_quic,\n", - " sum(ma.is_ip6_quic) as local_is_ip6_quic\n", + " sum(ma.is_ip4_tcp) as local_ip4_tcp_cnt, \n", + " sum(ma.is_ip6_tcp) as local_ip6_tcp_cnt, \n", + " sum(ma.is_ip4_quic) as local_ip4_quic_cnt,\n", + " sum(ma.is_ip6_quic) as local_ip6_quic_cnt\n", " FROM\n", " hole_punch_results hpr\n", " INNER JOIN multi_addresses_sets mas ON mas.id=hpr.listen_multi_addresses_set_id\n", @@ -429,459 +195,257 @@ " GROUP BY hpr.id, hpr.outcome, hpr.protocol_filters\n", " ) hpr\n", " INNER JOIN (\n", - " SELECT \n", - " hpa.hole_punch_result_id, \n", - " sum(ma.is_ip4_tcp) as is_ip4_tcp, \n", - " sum(ma.is_ip6_tcp) as is_ip6_tcp, \n", - " sum(ma.is_ip4_quic) as is_ip4_quic,\n", - " sum(ma.is_ip6_quic) as is_ip6_quic\n", - " FROM hole_punch_attempt hpa\n", - " INNER JOIN hole_punch_attempt_x_multi_addresses hpaxma on hpa.id = hpaxma.hole_punch_attempt\n", - " INNER JOIN temp_multiaddresses ma on hpaxma.multi_address_id = ma.id\n", - " WHERE ma.is_public AND NOT ma.is_relay\n", - " GROUP BY hpa.hole_punch_result_id\n", - " ) hpa ON hpa.hole_punch_result_id=hpr.id\n", + " SELECT \n", + " result_id,\n", + " sum(is_ip4_tcp) as ip4_tcp_cnt,\n", + " sum(is_ip6_tcp) as ip6_tcp_cnt,\n", + " sum(is_ip4_quic) as ip4_quic_cnt,\n", + " sum(is_ip6_quic) as ip6_quic_cnt\n", + " FROM (\n", + " SELECT DISTINCT\n", + " hpa.hole_punch_result_id as result_id, \n", + " ma.id as aid,\n", + " ma.is_ip4_tcp as is_ip4_tcp,\n", + " ma.is_ip6_tcp as is_ip6_tcp,\n", + " ma.is_ip4_quic as is_ip4_quic,\n", + " ma.is_ip6_quic as is_ip6_quic\n", + " FROM hole_punch_attempt hpa\n", + " INNER JOIN hole_punch_attempt_x_multi_addresses hpaxma on hpa.id = hpaxma.hole_punch_attempt\n", + " INNER JOIN temp_multiaddresses ma on hpaxma.multi_address_id = ma.id\n", + " WHERE ma.is_public AND NOT ma.is_relay\n", + " ) remote\n", + " GROUP BY remote.result_id\n", + " ) remote_addrs ON remote_addrs.result_id=hpr.id\n", " )\n", " GROUP BY \n", " outcome, \n", " protocol_filters, \n", - " local_is_ip4_tcp,\n", - " local_is_ip6_tcp,\n", - " local_is_ip4_quic,\n", - " local_is_ip6_quic,\n", - " remote_is_ip4_tcp,\n", - " remote_is_ip6_tcp,\n", - " remote_is_ip4_quic,\n", - " remote_is_ip6_quic\n", + " local_ip4_tcp_cnt,\n", + " local_ip6_tcp_cnt,\n", + " local_ip4_quic_cnt,\n", + " local_ip6_quic_cnt,\n", + " remote_ip4_tcp_cnt,\n", + " remote_ip6_tcp_cnt,\n", + " remote_ip4_quic_cnt,\n", + " remote_ip6_quic_cnt\n", "\"\"\"\n", - "df = pd.read_sql(final_results_q, con=conn)\n", - "df" + "df = pd.read_sql(final_results_q, con=conn)" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "fd2f445d", + "execution_count": 289, + "id": "fe9d2e5e", "metadata": {}, "outputs": [], "source": [ "dff = df" ] }, + { + "cell_type": "markdown", + "id": "04abf105", + "metadata": {}, + "source": [ + "add a string protocol filter column for easy grouping" + ] + }, + { + "cell_type": "code", + "execution_count": 306, + "id": "2bf0d538", + "metadata": {}, + "outputs": [], + "source": [ + "def pf_to_string(x):\n", + " if not x['protocol_filters']:\n", + " return \"No Filter\"\n", + " else:\n", + " ip_filter = 4 if x['protocol_filters'][0] == 4 else 6\n", + " transport_filter = \"tcp\" if x['protocol_filters'][1] == 6 else \"quic\"\n", + " return f\"ip:{ip_filter}|transport:{transport_filter}\"\n", + "dff['pf'] = dff.apply(lambda x: pf_to_string(x), axis=1)\n" + ] + }, + { + "cell_type": "markdown", + "id": "98e89c2d", + "metadata": {}, + "source": [ + "There is some issue with multiple ip4 addresses on remote side but this behaviour is not observed with ip6 cases. So whatever is causing the ip6 errors is something else. " + ] + }, { "cell_type": "code", - "execution_count": 12, - "id": "6345362c", + "execution_count": 300, + "id": "20d496ce", "metadata": {}, "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
outcomeprotocol_filterslocal_is_ip4_tcplocal_is_ip6_tcplocal_is_ip4_quiclocal_is_ip6_quicremote_is_ip4_tcpremote_is_ip6_tcpremote_is_ip4_quicremote_is_ip6_quiccountpf
0NO_STREAM[]000100201No Filter
1NO_STREAM[]000100601No Filter
2NO_STREAM[]000110001No Filter
3NO_STREAM[]000110102No Filter
4NO_STREAM[]000120001No Filter
.......................................
20727SUCCESS[41, 460]301021201ip:6|transport:quic
20728SUCCESS[41, 460]301040001ip:6|transport:quic
20729SUCCESS[41, 460]302010101ip:6|transport:quic
20730SUCCESS[41, 460]302020001ip:6|transport:quic
20731SUCCESS[41, 460]312010101ip:6|transport:quic
\n", - "

20732 rows × 12 columns

\n", - "
" - ], "text/plain": [ - " outcome protocol_filters local_is_ip4_tcp local_is_ip6_tcp \\\n", - "0 NO_STREAM [] 0 0 \n", - "1 NO_STREAM [] 0 0 \n", - "2 NO_STREAM [] 0 0 \n", - "3 NO_STREAM [] 0 0 \n", - "4 NO_STREAM [] 0 0 \n", - "... ... ... ... ... \n", - "20727 SUCCESS [41, 460] 3 0 \n", - "20728 SUCCESS [41, 460] 3 0 \n", - "20729 SUCCESS [41, 460] 3 0 \n", - "20730 SUCCESS [41, 460] 3 0 \n", - "20731 SUCCESS [41, 460] 3 1 \n", - "\n", - " local_is_ip4_quic local_is_ip6_quic remote_is_ip4_tcp \\\n", - "0 0 1 0 \n", - "1 0 1 0 \n", - "2 0 1 1 \n", - "3 0 1 1 \n", - "4 0 1 2 \n", - "... ... ... ... \n", - "20727 1 0 2 \n", - "20728 1 0 4 \n", - "20729 2 0 1 \n", - "20730 2 0 2 \n", - "20731 2 0 1 \n", - "\n", - " remote_is_ip6_tcp remote_is_ip4_quic remote_is_ip6_quic count \\\n", - "0 0 2 0 1 \n", - "1 0 6 0 1 \n", - "2 0 0 0 1 \n", - "3 0 1 0 2 \n", - "4 0 0 0 1 \n", - "... ... ... ... ... \n", - "20727 1 2 0 1 \n", - "20728 0 0 0 1 \n", - "20729 0 1 0 1 \n", - "20730 0 0 0 1 \n", - "20731 0 1 0 1 \n", - "\n", - " pf \n", - "0 No Filter \n", - "1 No Filter \n", - "2 No Filter \n", - "3 No Filter \n", - "4 No Filter \n", - "... ... \n", - "20727 ip:6|transport:quic \n", - "20728 ip:6|transport:quic \n", - "20729 ip:6|transport:quic \n", - "20730 ip:6|transport:quic \n", - "20731 ip:6|transport:quic \n", - "\n", - "[20732 rows x 12 columns]" + "pf outcome \n", + "ip:4|transport:tcp CONNECTION_REVERSED 289\n", + " FAILED 49471\n", + " NO_STREAM 930\n", + " SUCCESS 34651\n", + "Name: count, dtype: int64" ] }, - "execution_count": 12, + "execution_count": 300, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def pf_to_string(x):\n", - " if not x['protocol_filters']:\n", - " return \"No Filter\"\n", - " else:\n", - " ip_filter = 4 if x['protocol_filters'][0] == 4 else 6\n", - " transport_filter = \"tcp\" if x['protocol_filters'][1] == 6 else \"quic\"\n", - " return f\"ip:{ip_filter}|transport:{transport_filter}\"\n", - "dff['pf'] = dff.apply(lambda x: pf_to_string(x), axis=1)\n", - "dff" + "dff.query('pf == \"ip:4|transport:tcp\" & local_ip4_tcp_cnt==1 & remote_ip4_tcp_cnt>1').groupby(['pf', 'outcome'])['count'].sum()\n" ] }, { "cell_type": "code", - "execution_count": 17, - "id": "f5fe889c", + "execution_count": 301, + "id": "9b03ad8e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "pf outcome \n", - "ip:6|transport:tcp CONNECTION_REVERSED 324\n", - " FAILED 216\n", - " NO_STREAM 200\n", - " SUCCESS 7739\n", + "ip:4|transport:tcp CONNECTION_REVERSED 1749\n", + " FAILED 57277\n", + " NO_STREAM 1673\n", + " SUCCESS 277045\n", "Name: count, dtype: int64" ] }, - "execution_count": 17, + "execution_count": 301, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dff.query('pf == \"ip:6|transport:tcp\" & local_is_ip6_tcp==1 & remote_is_ip6_tcp==1').groupby(['pf', 'outcome'])['count'].sum()" + "dff.query('pf == \"ip:4|transport:tcp\" & local_ip4_tcp_cnt==1 & remote_ip4_tcp_cnt==1').groupby(['pf', 'outcome'])['count'].sum()\n" ] }, { "cell_type": "code", - "execution_count": 26, - "id": "012385d4", + "execution_count": 302, + "id": "ff1c607a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "pf outcome \n", - "ip:6|transport:tcp CONNECTION_REVERSED 64\n", - " FAILED 21194\n", - " NO_STREAM 142\n", - " SUCCESS 1744\n", + "pf outcome \n", + "ip:4|transport:quic CONNECTION_REVERSED 1137\n", + " FAILED 61653\n", + " NO_STREAM 1399\n", + " SUCCESS 66183\n", "Name: count, dtype: int64" ] }, - "execution_count": 26, + "execution_count": 302, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dff.query('pf == \"ip:6|transport:tcp\" & local_is_ip6_tcp==1 & remote_is_ip6_tcp>1').groupby(['pf', 'outcome'])['count'].sum()" + "dff.query('pf == \"ip:4|transport:quic\" & local_ip4_quic_cnt==1 & remote_ip4_quic_cnt>1').groupby(['pf', 'outcome'])['count'].sum()\n" ] }, { "cell_type": "code", - "execution_count": 28, - "id": "036f7904", + "execution_count": 303, + "id": "dd5c5fa3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "pf outcome \n", - "ip:4|transport:tcp CONNECTION_REVERSED 1445\n", - " FAILED 792\n", - " NO_STREAM 857\n", - " SUCCESS 266724\n", + "pf outcome \n", + "ip:4|transport:quic CONNECTION_REVERSED 1571\n", + " FAILED 53468\n", + " NO_STREAM 1270\n", + " SUCCESS 192483\n", "Name: count, dtype: int64" ] }, - "execution_count": 28, + "execution_count": 303, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dff.query('pf == \"ip:4|transport:tcp\" & local_is_ip4_tcp==1 & remote_is_ip4_tcp==1').groupby(['pf', 'outcome'])['count'].sum()" + "dff.query('pf == \"ip:4|transport:quic\" & local_ip4_quic_cnt==1 & remote_ip4_quic_cnt==1').groupby(['pf', 'outcome'])['count'].sum()\n" + ] + }, + { + "cell_type": "markdown", + "id": "087e9f48", + "metadata": {}, + "source": [ + "For ip6 tcp most of the errors happen when both sides had 1 address" ] }, { "cell_type": "code", - "execution_count": 25, - "id": "ab4b39c4", + "execution_count": 304, + "id": "9b31b657", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "pf outcome \n", - "ip:6|transport:quic CONNECTION_REVERSED 66\n", - " FAILED 30233\n", - " NO_STREAM 192\n", - " SUCCESS 2102\n", + "pf outcome \n", + "ip:6|transport:tcp CONNECTION_REVERSED 364\n", + " FAILED 20830\n", + " NO_STREAM 333\n", + " SUCCESS 8732\n", "Name: count, dtype: int64" ] }, - "execution_count": 25, + "execution_count": 304, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dff.query('pf == \"ip:6|transport:quic\" & local_is_ip6_quic==1 & remote_is_ip6_quic>1').groupby(['pf', 'outcome'])['count'].sum()" + "dff.query('pf == \"ip:6|transport:tcp\" & local_ip6_tcp_cnt==1 & remote_ip6_tcp_cnt==1').groupby(['pf', 'outcome'])['count'].sum()\n" ] }, { "cell_type": "code", - "execution_count": 62, - "id": "5d0e2488", + "execution_count": 305, + "id": "e8232b36", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "pf outcome \n", - "ip:4|transport:quic CONNECTION_REVERSED 2718\n", - " FAILED 116924\n", - " NO_STREAM 2705\n", - " SUCCESS 259798\n", + "pf outcome \n", + "ip:6|transport:tcp CONNECTION_REVERSED 24\n", + " FAILED 580\n", + " NO_STREAM 9\n", + " SUCCESS 751\n", "Name: count, dtype: int64" ] }, - "execution_count": 62, + "execution_count": 305, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dff.query('pf == \"ip:4|transport:quic\" & local_is_ip4_quic==1 & remote_is_ip4_quic==1').groupby(['pf', 'outcome'])['count'].sum()" + "dff.query('pf == \"ip:6|transport:tcp\" & local_ip6_tcp_cnt==1 & remote_ip6_tcp_cnt>1').groupby(['pf', 'outcome'])['count'].sum()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cd05dfa", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {