From 65f0a7e0e4a02c70ea6316d2df5a85bf2127f3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Mon, 13 Nov 2023 17:14:53 +0100 Subject: [PATCH] feat: find all SPs participating in FIL+ LDN (#1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- generated/ldn-providers.csv | 1076 ++++++++++++++++++++++++++++++++ package-lock.json | 14 + package.json | 1 + scripts/fetch-ldn-providers.js | 71 +++ 4 files changed, 1162 insertions(+) create mode 100644 generated/ldn-providers.csv create mode 100644 package-lock.json create mode 100644 scripts/fetch-ldn-providers.js diff --git a/generated/ldn-providers.csv b/generated/ldn-providers.csv new file mode 100644 index 0000000..1196489 --- /dev/null +++ b/generated/ldn-providers.csv @@ -0,0 +1,1076 @@ +f02661434 +f02639840 +f02364797 +f02122422 +f02122421 +f01946597 +f02038731 +f02760608 +f02814861 +f01972306 +f01986836 +f02053051 +f02760690 +f01967761 +f02843102 +f01926691 +f02061508 +f02101560 +f02101558 +f02841329 +f02519046 +f01975356 +f01980993 +f02828410 +f01972574 +f01930008 +f02809336 +f02366250 +f02056774 +f02059734 +f02100522 +f02055329 +f02055289 +f02057831 +f02516933 +f02366466 +f01990536 +f02230460 +f02055520 +f02366251 +f02366252 +f02519404 +f02031234 +f02007260 +f02635780 +f02204322 +f02010854 +f01131298 +f01938503 +f02211290 +f01952819 +f02060009 +f02046973 +f01963990 +f02365210 +f02080931 +f02369688 +f02036313 +f02833886 +f02814683 +f02814601 +f02521732 +f01940691 +f02831645 +f01945297 +f02088809 +f02831493 +f02831491 +f02057451 +f02130271 +f01859221 +f02829774 +f02814986 +f02820583 +f02820811 +f02354405 +f01878897 +f01919550 +f02198514 +f02056773 +f02369897 +f02069721 +f02067823 +f02637820 +f02828392 +f02097088 +f02815017 +f02347452 +f02280874 +f02824503 +f02824502 +f02220692 +f02100534 +f02224806 +f01935137 +f02364828 +f01914993 +f02224404 +f02046763 +f02807017 +f02147290 +f02100549 +f02068047 +f01949291 +f02085654 +f02100517 +f02002177 +f02816091 +f01880278 +f02204304 +f02204302 +f02060254 +f02814875 +f02192013 +f01966506 +f02759235 +f02056762 +f02366471 +f02144497 +f02044678 +f02226521 +f01151139 +f02216069 +f02213534 +f02252093 +f01945780 +f02809321 +f01936560 +f02100532 +f01549256 +f01905142 +f02046727 +f02122239 +f02046762 +f02063173 +f02122232 +f02078342 +f02117384 +f02122226 +f02066225 +f02771010 +f02056771 +f02191122 +f02122219 +f02087331 +f02027165 +f02078336 +f02367308 +f02089532 +f02231177 +f02239382 +f01941751 +f02101241 +f02063175 +f02519059 +f02227060 +f02366919 +f02032032 +f02191986 +f02361225 +f02368443 +f02130972 +f01917316 +f02029885 +f02055509 +f02092888 +f02057848 +f02057846 +f02366385 +f01687339 +f01512680 +f02057417 +f02126110 +f01982219 +f02223047 +f02100529 +f01999075 +f02224283 +f02251492 +f02224853 +f01934170 +f02038595 +f02204320 +f02363848 +f02035501 +f02060068 +f02031033 +f02362578 +f02362577 +f02204621 +f02204321 +f02252423 +f02100670 +f02130652 +f02130971 +f02361232 +f02358810 +f02361227 +f02359489 +f02130992 +f02281895 +f02053043 +f02055594 +f02227756 +f02204248 +f02063186 +f02170643 +f02239467 +f02257208 +f02100521 +f02147046 +f02346456 +f02147020 +f02274463 +f02182144 +f02063337 +f02041790 +f02237775 +f02090659 +f02307019 +f02026357 +f01997810 +f02036284 +f02059727 +f02020784 +f02126127 +f02191291 +f02060395 +f02247041 +f02059863 +f02257620 +f02041775 +f02208492 +f02190348 +f02057819 +f02253024 +f02056775 +f02145086 +f02036330 +f02036333 +f02037722 +f02105063 +f02059041 +f02208850 +f01870062 +f02246125 +f02181218 +f02123623 +f01870228 +f02048851 +f02182140 +f02195553 +f02239804 +f02034523 +f02096010 +f02199133 +f02208521 +f01972771 +f02034522 +f02066711 +f02168993 +f02185055 +f02057942 +f02057946 +f02057948 +f02203245 +f02117380 +f02123086 +f02126804 +f02057947 +f02041769 +f02185203 +f02117382 +f02038606 +f01945350 +f02191439 +f02092956 +f02100442 +f02132554 +f01916544 +f02185204 +f02129072 +f02207683 +f01984832 +f02197473 +f02223359 +f02128613 +f02128499 +f02128612 +f02126811 +f02126809 +f02190848 +f02057935 +f02060396 +f01986242 +f02208908 +f02129071 +f02098873 +f02052547 +f02048944 +f02216057 +f01984873 +f02123087 +f01984836 +f02063280 +f02047527 +f02093250 +f01880195 +f01724439 +f02207831 +f02094052 +f02051608 +f02010433 +f02075011 +f02093475 +f02093270 +f02198659 +f01794776 +f02171973 +f02057959 +f02057953 +f02036332 +f01940517 +f02206606 +f02146744 +f01984874 +f02093000 +f02067891 +f02146226 +f02204313 +f02126028 +f02036989 +f02046765 +f02129827 +f01752621 +f02199212 +f01913737 +f02090474 +f02057351 +f02057350 +f02114878 +f02101141 +f01989065 +f02047240 +f02192409 +f02010430 +f02041776 +f02191977 +f02094395 +f02130651 +f02088731 +f02048079 +f02084556 +f02130986 +f01880198 +f02047400 +f02047237 +f02052818 +f02094281 +f01941185 +f02186229 +f02186222 +f02186039 +f02179305 +f02062395 +f01697121 +f02099570 +f02085398 +f02060232 +f02010434 +f01984875 +f02182391 +f02182390 +f02030430 +f01985702 +f02115041 +f02025826 +f02047239 +f02129070 +f02064051 +f01074655 +f01884194 +f02082762 +f02138500 +f02100520 +f02100381 +f02014189 +f01987838 +f02041789 +f02041788 +f02047227 +f02047229 +f02047228 +f02084354 +f01992623 +f02003085 +f02125907 +f02125905 +f02124980 +f02067790 +f02123081 +f02122989 +f02047374 +f02118019 +f02051736 +f02011750 +f02011870 +f02024424 +f02114974 +f02052551 +f02055303 +f02031177 +f01943355 +f02011204 +f02104757 +f01980913 +f01986859 +f02101308 +f02085468 +f02025768 +f02100525 +f02100524 +f02024453 +f02090112 +f01972773 +f02059051 +f02038588 +f01941329 +f01901637 +f01966871 +f02095538 +f02095390 +f02095387 +f02095364 +f01974601 +f02094419 +f02063152 +f0215074 +f02066000 +f01992841 +f01379794 +f01752758 +f01974717 +f01974716 +f02085938 +f02068015 +f02081703 +f01988795 +f02047965 +f01996948 +f01987786 +f02024329 +f02062133 +f02057490 +f02060125 +f02059052 +f01837713 + +f01990040 +f02191277 +f01955887 +f01880278 +f02191274 +f02071888 +f01174487 +f02049082 +f02364586 +f02104443 +f02063722 +f01999562 +f02058781 +f02257197 +f0700600 +f01924789 +f02126522 +f02130336 +f02208630 +f01993419 +f01076539 +f01986858 +f02063792 +f02026054 +f02026280 +f02027248 +f01981292 +f01908773 +f02231177 +f02045753 +f02044678 +f02031234 +f02030500 +f02029885 +f01549256 +f02036330 +f02010430 +f02011750 +f02026357 +f01992879 +f01876344 +f01917316 +f02010315 +f01988795 +f01880196 +f02050289 +f02024453 +f02011870 +f01941185 +f02049279 +f01870062 +f02060232 +f02038588 +f01997810 +f02010433 +f02010434 +f02059041 +f02048079 +f01900756 +f01920758 +f01992841 +f01966506 +f02002405 +f01907539 +f0215074 +f01982040 +f01916279 +f02057819 + +f01943355 +f02057451 +f01916544 +f01512680 +f01981070 +f01945780 +f01876955 +f01993419 +f02055596 +f02055594 +f01985642 +f01985585 +f02051608 +f02024424 +f02025826 +f01972571 +f01966871 +f02055329 +f02002177 +f02055289 +f01984693 +f01131298 +f01880278 +f02052818 +f01972773 +f02006102 +f02053051 +f02053043 +f01996948 +f01919298 +f01857952 +f01925065 +f01926691 +f01884194 +f01985718 +f02052551 +f02052547 +f02048944 +f02046727 +f01074655 +f02051736 +f02051580 +f01967761 +f02014189 +f01914993 +f01930008 +f02009588 +f02048851 +f01880195 +f01379794 +f01999816 +f01913737 +f01980914 +f02010854 +f01965238 +f01837711 +f02047527 +f01854901 +f02047378 +f02047240 +f02047239 +f01974716 +f02047237 +f02047231 +f02047229 +f02047228 +f02047227 +f01936354 +f01924649 +f01987869 +f01943349 +f01512996 +f02041789 +f02041790 +f02041788 +f02041777 +f02041776 +f02041775 +f02041769 +f0397376 +f01917191 +f02038556 +f02038555 +f02038351 +f02029897 +f02037722 +f01870228 +f02036333 +f02036332 +f02036313 +f02036306 +f02036284 +f01972574 +f02034522 +f02034523 +f01941237 +f02035271 +f02003085 +f01837713 +f01974601 +f01974717 +f01992623 +f02031177 +f02031033 +f01945350 +f01901107 +f01941184 +f01998998 +f01987838 +f01880198 +f01787692 +f02026234 +f02025768 +f01863832 +f01924388 +f02022967 +f02020784 +f02011204 +f01941329 +f01994528 +f02006894 +f02006103 +f01972306 +f01972309 +f01881929 +f01667796 +f01945167 +f01716661 +f01924563 +f01919550 +f01916905 +f01995046 +f01916961 +f01993430 +f01980990 +f01973453 +f01986859 +f01905314 +f01916793 +f01986242 +f01940517 +f01949291 +f01972771 +f01938462 +f01920820 +f01993555 +f01869693 +f01890407 +f01923772 +f01920136 +f01985702 +f02006097 +f01941183 +f01937552 +f02005507 +f01993597 +f02005305 +f02005299 +f02005295 +f01993594 +f01789840 +f01878897 +f01999116 +f01924648 +f01849553 +f01986979 +f01998638 +f01905142 +f01908314 +f01997841 +f01972207 +f01940441 +f01973010 +f01946657 +f01935137 +f01985682 +f01471028 +f01994024 +f01924664 +f01975371 +f01889672 +f01992627 +f01917354 +f01986465 +f01948957 +f01940691 +f01884874 +f01952819 +f01929879 +f01930007 +f01941584 +f01706781 +f01989121 +f01921402 +f01904769 +f01937484 +f01943532 +f01762699 +f01941186 +f01987477 +f01945035 +f01941690 +f01987325 +f01942130 +f01904026 +f01986884 +f01986836 +f01965327 +f01884576 +f01785922 +f01904773 +f01926620 +f01921336 +f01985891 +f01300307 +f01973609 +f01968227 +f01985575 +f01984645 +f01984555 +f01946597 +f01981702 +f01916263 +f01981679 +f01836473 +f01975356 +f01923357 +f01944321 +f01907516 +f01765178 +f01945297 +f01980993 +f01819389 +f01930006 +f01896580 +f01975665 +f01861175 +f01890244 +f01948527 +f01069489 +f01876916 +f01938396 +f01902002 +f01855808 +f01905508 +f01927870 +f01687339 +f01972749 +f01901637 +f01947630 +f01880006 +f01952539 +f01885335 +f01673159 +f01936823 +f01941015 +f01944211 +f01888096 +f01860775 +f01902528 +f01513502 +f01940667 +f01848528 +f01911398 +f01900762 +f01877976 +f01925027 +f01934115 +f01853599 +f01843149 +f01919335 +f01512103 +f01948518 +f01899380 +f01903406 +f01619806 +f01964269 +f01769561 +f01963990 +f01923695 +f01903807 +f01941661 +f01788118 +f01843336 +f01788133 +f01876957 +f01941684 +f01788131 +f01917163 +f01917209 +f01859966 +f01912619 +f01788458 +f01415525 +f01358347 +f01904627 +f01882154 +f01912635 +f01788135 +f01912885 +f01868373 +f01893725 +f01885054 +f01451206 +f01839961 +f01880887 +f01513492 +f01929808 +f01904436 +f01788115 +f01181090 +f01885668 +f01856640 +f01896674 +f01513499 +f01885050 +f01614121 +f01909103 +f01841978 +f01513006 +f01863807 +f01771053 +f01886305 +f022341 +f01310929 +f01917492 +f01881217 +f01103201 +f0743060 +f01837712 +f01623954 +f01898024 +f01513503 +f01872918 +f01513013 +f01852534 +f01861433 +f01302553 +f01513510 +f01881688 +f01358352 +f01752086 +f01513008 +f01906730 +f01850099 +f01513012 +f01862947 +f01837708 +f01860990 +f01841152 +f01513505 +f01705505 +f01872946 +f01901063 +f01513005 +f01358346 +f01817237 +f01898447 +f01666929 +f01841335 +f01610897 +f01893732 +f01889677 +f01614579 +f01600628 +f01682491 +f01512500 +f01885042 +f01705503 +f01815858 +f0164878 +f01879384 +f01878028 +f01877027 +f01876942 +f01659102 +f01729611 +f01870661 +f01381852 +f01872489 +f01878945 +f01880562 +f01872319 +f01872050 +f01878721 +f01878854 +f01879782 +f01879976 +f01940945 +f01870057 +f01857951 +f01873934 +f01864590 +f01513012 +f01513005 +f01885778 +f01859221 +f01844774 +f01859762 +f01513013 +f01851817 +f01379794 +f01513006 +f01860352 +f01851513 +f01513009 +f01513499 +f01513510 +f01819389 +f01765178 +f01820158 +f01841152 +f01837711 +f01837708 +f01837712 +f01837713 +f01819267 +f01851738 +f01859736 +f01801993 +f01820352 +f01828472 +f01828473 +f01820341 +f01859735 +f01839447 +f01820335 +f01820329 +f01878897 +f01817237 +f01817238 +f01815858 +f01835101 +f01513503 +f01513502 +f01513492 +f01787692 +f01615359 +f01687339 +f01729602 +f01729611 +f01715290 +f01656506 +f01656452 +f01513007 +f01666929 +f01513003 +f01752082 +f01724421 +f01752758 +f01512500 +f01706785 +f01705505 +f01724439 +f01706781 +f01705499 +f01752621 +f01683646 +f01404557 +f01673159 +f01659103 +f01653686 +f01659102 +f01610839 +f01611135 +f01659109 +f01667796 +f01619806 +f01615359 +f01614121 +f01614579 +f01513008 +f01916303 +f01621962 +f01623954 +f01624861 +f01752086 +f01600628 +f01676203 +f01610897 +f01512103 +f01069489 +f01524687 +f01876551 +f01512995 +f01705503 +f01697121 +f01566819 +f01358347 +f01471028 +f01611268 +f01484077 +f0215074 +f01451206 +f01345229 +f01139913 +f01916279 +f01358352 +f01310929 +f01302553 +f01034818 +f01385053 +f01404557 +f022341 +f01166068 +f01358346 +f01020500 +f0985604 +f01198173 +f01356431 +f0743060 +f01356432 +f01131298 +f01151139 +f0421499 +f0843755 +f01131298 +f01017847 +f01251071 +f0856505 +f01224308 +f01103201 +f01549256 +f0700600 +f0843755 +f01037760 +f01181090 +f0985604 +f01127441 +f01221878 +f01343523 +f01017847 +f01705501 +f01451238 +f01345229 +f01166068 +f01139913 +f01343509 +f01381852 +f01705504 +f0397376 +f0700600 +f0421499 +f01224308 +f01251071 +f01037760 +f01151139 +f0397376 +f0856505 diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..49788b1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,14 @@ +{ + "name": "fil-deal-ingester", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "fil-deal-ingester", + "version": "0.0.1", + "license": "(Apache-2.0 AND MIT)", + "devDependencies": {} + } + } +} diff --git a/package.json b/package.json index 46b0cb8..66e311a 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "fil-deal-ingester", "private": true, "version": "0.0.1", + "type": "module", "repository": { "type": "git", "url": "git+https://github.com/filecoin-station/fil-deal-ingester.git" diff --git a/scripts/fetch-ldn-providers.js b/scripts/fetch-ldn-providers.js new file mode 100644 index 0000000..f0803d8 --- /dev/null +++ b/scripts/fetch-ldn-providers.js @@ -0,0 +1,71 @@ +import fs from 'node:fs/promises' +import { dirname, resolve, relative } from 'node:path' +import { fileURLToPath } from 'node:url' +import { setTimeout } from 'node:timers/promises' + +const DATACAPS_URL = 'https://api.datacapstats.io/public/api/' +const API_KEY = process.env.API_KEY ?? await getApiKey() + +const allLdnProviders = [] + +const notaries = await findNotaries('ldn') +console.log('Found %s notaries', notaries.length) + +for (const notaryAddressId of notaries) { + const providers = await getVerifiedClientsOfNotary(notaryAddressId) + console.log('Notary %s tracks %s providers', notaryAddressId, providers.length) + allLdnProviders.push(...providers) + // slow down to avoid hitting rate limits + await setTimeout(100) +} + +console.log('Found %s FIL+ LDN providers in total', allLdnProviders.length) +const outfile = resolve(dirname(fileURLToPath(import.meta.url)), '../generated/ldn-providers.csv') +await fs.writeFile(outfile, allLdnProviders.map(p => `${p}\n`).join('')) +console.log('The list was written to %s', relative(process.cwd(), outfile)) + +/** @returns {Promise} */ +async function getApiKey () { + const res = await fetch(`${DATACAPS_URL}/getApiKey`) + if (!res.ok) { + throw new Error(`Cannot obtain DataCapStats API key: ${res.status}\n${await res.text()}`) + } + + return await res.text() +} + +/** @returns {Promise} */ +async function findNotaries (filter) { + const res = await fetch( + buildUrlWithQueryString('getVerifiers', { limit: 1000, filter }), + { headers: { 'X-API-KEY': API_KEY } } + ) + + if (!res.ok) { + throw new Error(`Cannot query notaries: ${res.status}\n${await res.text()}`) + } + + const body = await res.json() + return body.data.map(obj => obj.addressId) +} + +/** @returns {Promise} */ +async function getVerifiedClientsOfNotary (notaryAddressId) { + const res = await fetch( + buildUrlWithQueryString(`getVerifiedClients/${notaryAddressId}`, { limit: 1000 }), + { headers: { 'X-API-KEY': API_KEY } } + ) + + if (!res.ok) { + throw new Error(`Cannot query verified clients: ${res.status}\n${await res.text()}`) + } + + const body = await res.json() + return body.data.map(obj => obj.addressId) +} + +function buildUrlWithQueryString (endpointPath, searchParams) { + const url = new URL(endpointPath, DATACAPS_URL) + url.search = new URLSearchParams(searchParams).toString() + return url +}