From 92412fd84eb76088d3aa5c53db2983cc4d9e11f5 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Fri, 24 May 2024 10:01:14 -0700
Subject: [PATCH 01/23] testing if github runner can scan 1k domains without
 crazy timeouts

---
 .github/workflows/ci.yml         |    4 +
 largescaleintegration/top-1k.csv | 1000 ++++++++++++++++++++++++++++++
 2 files changed, 1004 insertions(+)
 create mode 100644 largescaleintegration/top-1k.csv

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 18b5791e..8a265b2a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,3 +37,7 @@ jobs:
           sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
           python --version
           ./integration_tests.py
+      - name: Large-Scale Integration Test
+        run: |
+          python --version
+          cat largescaleintegration/top-1k.csv | ./zdns A --iterative | grep "TIMEOUT" | wc -l
diff --git a/largescaleintegration/top-1k.csv b/largescaleintegration/top-1k.csv
new file mode 100644
index 00000000..8d758930
--- /dev/null
+++ b/largescaleintegration/top-1k.csv
@@ -0,0 +1,1000 @@
+m.fabswingers.com
+www.dmm.co.jp
+anoboy.baby
+www.cmoa.jp
+visortmo.com
+shahvani.com
+sattaamatka.co.com
+www.asos.com
+arca.live
+letterboxd.com
+www.ebay.com
+www.bfmtv.com
+www.rat.xxx
+my.xhtab3.com
+www.flashscore.com
+cz.pornhub.com
+studio.code.org
+www.t-online.de
+accounts.google.com
+asuratoon.com
+glife.bingoplus.com
+www.tiktok.com
+aniwave.to
+schools.madrasati.sa
+gall.dcinside.com
+m.fmkorea.com
+mangabuddy.com
+thisvid.com
+www.flipkart.com
+sp.oddspark.com
+www.fmkorea.com
+www.porn300.com
+context.reverso.net
+ijavhd.com
+www.mobile.de
+www.sexindrag.com
+de.m.wikipedia.org
+exhentai.org
+onlyindianx.cc
+secure.bankofamerica.com
+www.mercadolibre.com.ar
+www.linkedin.com
+detail.chiebukuro.yahoo.co.jp
+fr.pornhub.com
+m.indiamart.com
+www.ebay.de
+www.facebook.com
+www.researchgate.net
+battwo.com
+web.whatsapp.com
+pikabu.ru
+myreadingmanga.info
+www.netflix.com
+www.samsung.com
+www.tokopedia.com
+medium.com
+ge.xhamster.com
+www.bbc.com
+www.publi24.ro
+olimpbet.kz
+www.polovniautomobili.com
+m.yahoo.co.jp
+www.coco.gg
+www.69shu.pro
+yako.red
+escortbabylon.net
+poczta.wp.pl
+www.dagbladet.no
+m.ss.com
+m.bild.de
+ria.ru
+www.dailymotion.com
+retail.onlinesbi.sbi
+myactivity.google.com
+nregastrep.nic.in
+www.aznude.com
+coinmarketcap.com
+www.gazzetta.it
+tv.simontokx.online
+character.ai
+pmkisan.gov.in
+mobile-tracker-free.com
+quizizz.com
+mov.ibomma.rs
+www.masrawy.com
+www.google.es
+as.com
+m.youtube.com
+bpexch.com
+betinexchange.com
+ncz3u7cj2.com
+www.jusbrasil.com.br
+auto.drom.ru
+www.subito.it
+poki.com
+newtoki330.com
+giris.turkiye.gov.tr
+www.protothema.gr
+18comic.vip
+m.olx.pl
+tvn24.pl
+m.naver.com
+www.cardmarket.com
+wordwall.net
+www.varzesh3.com
+www.mynet.com
+es.aliexpress.com
+www.hentai.name
+www.hdhindisex.com
+www.netshoes.com.br
+wl4.peachavocado.com
+hackerdoslot.com
+www.hln.be
+xhdepot.site
+xvideosporno.blog.br
+czbooks.net
+genius.com
+ekstrabladet.dk
+launchpad.classlink.com
+danbooru.donmai.us
+www.action.com
+onlar.az
+doublelist.com
+ja.wikipedia.org
+news.livedoor.com
+www.anses.gob.ar
+mlbpark.donga.com
+support.google.com
+www.kidsa-z.com
+www.merkur.de
+accounts.spotify.com
+index.hu
+bollyflix.nexus
+www.hindustantimes.com
+web.spaggiari.eu
+scholar.google.com
+www.bestbuy.com
+www.quora.com
+www.instagram.com
+www.avito.ru
+www.sportskeeda.com
+www.xnxx.tv
+g1.globo.com
+www.daily.co.jp
+play.blooket.com
+simpcity.su
+www.kamababa.desi
+www.starfall.com
+nz.ua
+saednews.com
+www.finn.no
+member.sss.gov.ph
+hianime.to
+www.mangago.me
+e621.net
+ar.m.wikipedia.org
+gdz.ru
+byjus.com
+www.viva.co.id
+www.vinted.fr
+www.google.com.tr
+www.betpawa.ug
+www.sciencedirect.com
+brainly.co.id
+www.amazon.eg
+www.mercadolivre.com.br
+www.google.com.hk
+www.indeed.com
+www.lanacion.com.ar
+nl.pornhub.com
+steamcommunity.com
+an1.com
+mangakakalot.com
+truyenfull.vn
+linktr.ee
+m.cafe.daum.net
+vnexpress.net
+www.tgju.org
+www.ukr.net
+betproexch.com
+www.spotify.com
+fr.xhamster.com
+www.sugotoku.docomo.ne.jp
+kolkataff.in
+www.patria.org.ve
+www.roblox.com
+bdvenlinea.banvenez.com
+mail.google.com
+www.google.de
+news.infoseek.co.jp
+www.amazon.es
+www.espncricinfo.com
+zh.m.wikipedia.org
+www.aajtak.in
+n.news.naver.com
+www.livescore.com
+www.remove.bg
+fa.m.wikipedia.org
+secure.livechatinc.com
+www.almasryalyoum.com
+www.ebay.co.uk
+synergia.librus.pl
+www.ninisite.com
+www.google.pl
+chromewebstore.google.com
+m.olx.ua
+s.kakaku.com
+wimt.confirmtkt.com
+www.globo.com
+www.disneyplus.com
+es.pornhub.com
+gelbooru.com
+servicioscorp.anses.gob.ar
+www.xvideos3.com
+www.irs.gov
+xbato.com
+salonboard.com
+tel.ibomma.nexus
+tw.stock.yahoo.com
+www.tjk.org
+newtoki331.com
+readmanga.live
+www.programme-tv.net
+br.pinterest.com
+www.keralalotterytoday.com
+xxxbfvideos.org
+escortalligator.com.listcrawler.eu
+www.jutarnji.hr
+www.namasha.com
+finance.yahoo.com
+missav.com
+www.joyclub.de
+www.ign.com
+www.amazon.in
+www.independent.co.uk
+www.ryanair.com
+mangaschan.net
+mangatoto.org
+www.goojara.to
+www.tradingview.com
+www.twitch.tv
+11xplay.pro
+elpais.com
+snaptik.app
+people.com
+m.humoruniv.com
+ru.xhamster.com
+xhwide4.com
+www.inven.co.kr
+ssstik.io
+sso.acesso.gov.br
+www.cdiscount.com
+www.betika.com
+enabiz.gov.tr
+sattasport.in
+xoso.com.vn
+xnxx.health
+www.ynet.co.il
+www.google.com
+sattamatkano1.me
+hanime1.me
+novel18.syosetu.com
+www.dailymail.co.uk
+www.lefigaro.fr
+www.casinoplus.com.ph
+www.slideshare.net
+e-hentai.org
+m.tribunnews.com
+www.jlpt-maja.org.mm
+olxmewah.com
+www.futbin.com
+www.lxtgame.com
+www.wattpad.com
+music-fa.com
+www.iltalehti.fi
+www.google.com.mx
+timesofindia.indiatimes.com
+www.youjizz.com
+fantia.jp
+tools.usps.com
+www.aemet.es
+www.boyfriendtv.com
+jable.tv
+www.24h.com.vn
+faphouse.com
+www.list.am
+hot-sex-tube.com
+www.hurriyet.com.tr
+noodlemagazine.com
+www.xnxx3.com
+satta-king-fixed-no.in
+calendar.google.com
+www.3bmeteo.com
+coomer.su
+m.hqporner.com
+www.apornvideo.com
+dewafilm.xyz
+smt.docomo.ne.jp
+twitter.com
+account.samsung.com
+archiveofourown.org
+vlxx.moe
+m.sexemodel.com
+www.imagefap.com
+www.ojogodobicho.com
+www.dogdrip.net
+www.google.it
+finance.yahoo.co.jp
+www.aparat.com
+moi.ibomma.re
+pl.xhamster.com
+www.idnes.cz
+xhamster42.desi
+kompoz2.com
+tel.ibomma.rsvp
+m.vk.com
+www.amazon.fr
+www.cnnindonesia.com
+en.luxuretv.com
+select.mamastar.jp
+pubmed.ncbi.nlm.nih.gov
+www.biblegateway.com
+www.marmiton.org
+dantri.com.vn
+s.mogg.ameba.jp
+www.nikkansports.com
+v.daum.net
+www.avto.net
+www.naver.com
+m.hollywoodbets.net
+m.ok.ru
+www.ndtv.com
+rajwap.cc
+search.yahoo.co.jp
+ya.ru
+www.foxnews.com
+www.marca.com
+www.youtube.com
+www.bing.com
+xxxbfvideos.net
+www.nettruyenff.com
+zistop.com
+zbporn.tv
+www.unotv.com
+www.amazon.com
+sxyprn.com
+theqoo.net
+khoahoc.vietjack.com
+namu.wiki
+www.fulldesisex.com
+schools.emaktab.uz
+outlook.live.com
+www.discogs.com
+de.pornhub.org
+apps.facebook.com
+old.reddit.com
+www.flashscore.fr
+www.leroymerlin.fr
+www.jw.org
+www.aliexpress.com
+www.scribd.com
+bokepindoh.sbs
+duckduckgo.com
+drive.usercontent.google.com
+ecnavi.jp
+nypost.com
+www.amazon.co.jp
+www.ladepeche.fr
+www.patreon.com
+xhamster.desi
+game.granbluefantasy.jp
+www.google.co.in
+tsn.ua
+www.sexvid.xxx
+udn.com
+la.spankbang.com
+ge.xhamster.desi
+tamilyogi.red
+telegram.org
+www.otomoto.pl
+www.cnbcindonesia.com
+rule34.xxx
+www.bpexch.com
+topmanhua.com
+www.imdb.com
+xhtab3.com
+m.haber7.com
+porno18.site
+www.wp.pl
+stackoverflow.com
+www.sikayetvar.com
+actu.fr
+item.rakuten.co.jp
+produto.mercadolivre.com.br
+litnet.com
+grapee.jp
+myapps.classlink.com
+www.agoda.com
+www.index.hr
+beauty.hotpepper.jp
+mbigpara.hurriyet.com.tr
+kemono.su
+www.staseraintv.com
+shinigami.cx
+www.ouest-france.fr
+direct.lc.chat
+m.shein.com
+www.vg.no
+www.w3schools.com
+esaj.tjsp.jus.br
+hiroba.dpoint.docomo.ne.jp
+www.amazon.com.tr
+www.ozon.ru
+lichess.org
+www.goodreads.com
+www.mathplayground.com
+www.nexusmods.com
+app.hubspot.com
+funpay.com
+m.kooora.com
+www.wordreference.com
+www.goal.com
+truyenqqvn.com
+uk.indeed.com
+www.fanpage.it
+www.lowes.com
+www.diretta.it
+search.naver.com
+www.leboncoin.fr
+m.ppomppu.co.kr
+www.dns-shop.ru
+www.metropoles.com
+chaturbate.com
+www.serviporno.com
+bato.to
+game8.jp
+alarbda.com
+xxxvideo.link
+www.pinterest.com
+www.vietjack.com
+ita.xhamster.com
+www.trendyol.com
+www.ebay.it
+vegamovies.rsvp
+www.forbes.com
+return.gst.gov.in
+www.jcpenney.com
+xn--l3cg7a8a0cwa3f.com
+www.realtor.com
+ameblo.jp
+iporntv.net
+www.google.fr
+chance.enjoy.point.auone.jp
+business.facebook.com
+m.chaturbate.com
+www.5movierulz.dog
+id.xhamster.com
+weather.com
+www.chegg.com
+www.ixxx.com
+www.sportybet.com
+auto.ria.com
+m.youm7.com
+www.amazon.de
+mpokec.azet.sk
+www.filmyzilla.com.ng
+www.jra.go.jp
+in3x.net
+www.bt.dk
+www.thesun.co.uk
+it.m.wikipedia.org
+manhwaclan.com
+screenrant.com
+trilltrill.jp
+www.jiocinema.com
+www.tvonenews.com
+login.caixa.gov.br
+finance.naver.com
+www.bakecaincontrii.com
+emaktab.uz
+movie.eroterest.net
+www.1mg.com
+www.eltiempo.es
+tr.m.wikipedia.org
+tmohentai.com
+www.seznam.cz
+auth.uber.com
+photos.google.com
+www.flashscore.com.br
+kuji.dmarket.docomo.ne.jp
+docs.google.com
+www.amazon.com.mx
+sites.google.com
+cricbet99.win
+www.ilovepdf.com
+www.skroutz.gr
+my.porno365.pics
+support.microsoft.com
+freeip.me
+www.turkiye.gov.tr
+sports.yahoo.co.jp
+spboss.blog
+www.mundodeportivo.com
+www.tokyomotion.net
+fast.com
+www.conectate.com.do
+yatube.net
+www.chordtela.com
+br.skokka.com
+komikindo.tv
+www.gov.br
+www.elmundo.es
+app2dn.fextok.com
+race.sp.netkeiba.com
+evexxx.com
+www.clarin.com
+www.20minutos.es
+thisav.com
+www.ptt.cc
+v22-ispdy5ersd-ai-verify--gcs.g-xyz.xyz
+sports.news.naver.com
+sellercentral.amazon.com
+www.espn.com
+www.baidu.com
+zozo.jp
+www.n-tv.de
+dzen.ru
+harimanga.com
+lms.lausd.net
+m.happymh.com
+www.booking.com
+yts.mx
+gimy.ai
+pixabay.com
+tenki.jp
+gshow.globo.com
+me.porno666.link
+doujindesu.tv
+schools.kundelik.kz
+www.reddit.com
+www.zillow.com
+filmyfly.pm
+www.idealista.com
+www.binance.com
+www.interia.pl
+in.search.yahoo.com
+in.pinterest.com
+poop.com.co
+www.redtube.com
+www.mayoclinic.org
+betnacional.com
+br.betano.com
+www.1024tera.com
+9animetv.to
+www.jalan.net
+bakusai.com
+ko.xhamster2.com
+hochi.news
+vk.com
+www.coingecko.com
+www.oricon.co.jp
+moviesda9.me
+www.ajio.com
+www.ancestry.com
+www.credit-agricole.fr
+hdhub4u.pm
+client.schwab.com
+ncode.syosetu.com
+m.999.md
+www.istockphoto.com
+bbs.ruliweb.com
+www.costco.com
+www.curseforge.com
+search.rakuten.co.jp
+www.cricbuzz.com
+www.creditkarma.com
+www.sex.com
+baseball.yahoo.co.jp
+www.123-hd.com
+www.msn.com
+550909.com
+cashier.hkppay.com
+imgsrc.ru
+manga168.com
+xxxnd.com
+hitomi.la
+www.pussyboy.net
+piccoma.com
+m.autoplius.lt
+www.shaalaa.com
+email.seznam.cz
+pay.icespay.com
+www.superporn.com
+th.spankbang.com
+cn.pornhub.com
+www.desitelugusex.com
+yako.net
+www.freepik.com
+www.makemytrip.com
+store.steampowered.com
+okxxx1.com
+www.yahoo.com
+www.justdial.com
+www.udemy.com
+www.bet365.com
+www.merriam-webster.com
+discord.com
+sp.mbga.jp
+www.russianfood.com
+m.fanfiction.net
+www.kurir.rs
+www.zearn.org
+pet.gree-apps.net
+www.amarujala.com
+www.fiverr.com
+link.coupang.com
+clever.com
+lotterysambadresult.in
+www.clip2vip.com
+www.mediaexpert.pl
+pl.m.wikipedia.org
+www.letras.mus.br
+www.google.ca
+ifilo.net
+myaccount.google.com
+www.fuq.com
+www.eenadu.net
+m.magazineluiza.com.br
+www.google.co.uk
+faq.whatsapp.com
+in.indeed.com
+moviesda9.co
+mail.yahoo.com
+t.me
+m.tabor.ru
+news.google.com
+m.gsmarena.com
+onlyfans.com
+www.bilibili.tv
+wl2.peachavocado.com
+portal.auone.jp
+unifiedportal-mem.epfindia.gov.in
+uidai.gov.in
+www.flashscore.pl
+www.google.com.br
+m.beboo.ru
+login.yahoo.com
+tubidy.cool
+mail.ru
+www.etsy.com
+www.mediafire.com
+www.ilmeteo.it
+pl.pornhub.com
+transit.yahoo.co.jp
+chat.openai.com
+newtoki329.com
+www.cronista.com
+psiphon.news
+member.rakuten-sec.co.jp
+g.laroza.net
+skipthegames.com
+apnaludo.com
+www.xvideos53.com
+www.nu.nl
+www.sozcu.com.tr
+www.pussyspace.com
+m.pantip.com
+www.deviantart.com
+www.promiedos.com.ar
+www.homedepot.com
+www.tvguia.es
+www.wildberries.ru
+wol.jw.org
+rezka.ag
+www.macys.com
+www.buzzfeed.com
+www.dropbox.com
+hi.xhamster42.desi
+www.adobe.com
+www.coolmathgames.com
+rule34video.com
+www.google.co.jp
+cookpad.com
+ser.brstej.com
+togetter.com
+animeyuzu.com
+www.cityheaven.net
+www.furaffinity.net
+www.nikkei.com
+game.hiroba.dpoint.docomo.ne.jp
+www.argos.co.uk
+www.accuweather.com
+tw.yahoo.com
+br.xhamster.com
+www.ahindian.com
+liquipedia.net
+onedrive.live.com
+tathya.uidai.gov.in
+www.idealista.it
+www.pixiv.net
+testing.illuminateed.com
+uranai.nosv.org
+wl5.peachavocado.com
+hentailib.me
+www.chip.de
+www.redfin.com
+hot.noodlemagazine.com
+boards.4chan.org
+ru.sex-studentki.cash
+tukif.com
+kakuyomu.jp
+topics.smt.docomo.ne.jp
+jut.su
+www.xvideos.es
+www.zalando.de
+aniwatchtv.to
+uchi.ru
+www.fsiblog3.club
+www.yahoo.co.jp
+xnxx.com.se
+www.prokerala.com
+scratch.mit.edu
+www.emag.ro
+www.hdtube.porn
+meet.google.com
+m.search.naver.com
+www.bbc.co.uk
+xxxsexjav.com
+ficbook.net
+www.qorno.com
+github.com
+www.ikea.com
+www.target.com
+www.news.com.au
+www.haberler.com
+archive.org
+fr.m.wikipedia.org
+news.yahoo.co.jp
+mechacomic.jp
+www.terabox.app
+www.5movierulz.blog
+www.whatsapp.com
+cuetug.ntaonline.in
+www.walmart.com
+mangalib.me
+www.cnn.com
+xhamster2.com
+m.dcinside.com
+www.lemonde.fr
+www.universal-credit.service.gov.uk
+www.ad.nl
+www.annunci69.it
+m.ruliweb.com
+www.livescore.in
+www.xvideos4.com
+beta.character.ai
+s.dto.jp
+xxx18hot.com
+es.xhamster.com
+www.pornid.name
+www.moneycontrol.com
+av19.gg
+www3.nhk.or.jp
+news.goo.ne.jp
+www.sexybluefilm.com
+in.bookmyshow.com
+www.autotrader.co.uk
+www.telegraaf.nl
+www.newsweek.com
+www.sahibinden.com
+www.myntra.com
+gemini.google.com
+girlschannel.net
+hivemapper.com
+mat6tube.com
+ge.globo.com
+brainly.in
+de.pornhub.com
+www.filmweb.pl
+homepage.vivo.com
+diziwatch.net
+imhentai.xxx
+www.paypal.com
+www.kompas.com
+syosetu.org
+www.apple.com
+supjav.com
+www.kohls.com
+toyhou.se
+lectormanga.lat
+www.francetvinfo.fr
+www.xnxx2.com
+bet.caliente.mx
+bbs.animanch.com
+quillbot.com
+www.blic.rs
+www.digikala.com
+www.thscore.mobi
+ads.google.com
+vegamovies.ong
+www.gov.uk
+www.chess.com
+www.onet.pl
+shopee.com.br
+www.olx.com.br
+www.ixl.com
+www.ixbt.com
+www.ncbi.nlm.nih.gov
+dmarket.docomo.ne.jp
+articulo.mercadolibre.com.ar
+m.facebook.com
+sp.jra.jp
+xosodaiphat.com
+www.rezultati.com
+rt.pornhub.com
+www.52shuku.vip
+www.colamanga.com
+www.youporn.com
+motherless.com
+www.betway.co.za
+fapello.com
+kkstories.com
+mbasic.facebook.com
+fatalmodel.com
+www.hltv.org
+m.imdb.com
+www.bilibili.com
+www.nike.com
+www.amazon.com.br
+www.isaimini.com.sb
+www.pornhat.one
+www.milliyet.com.tr
+pfms.nic.in
+play.google.com
+www.livehindustan.com
+www.wannonce.com
+portal.librus.pl
+xhamster1.desi
+luxmovies.biz
+ru.m.wikipedia.org
+dpboss.services
+www.sexvid.pro
+xnxxcom.xyz
+www.croxyproxy.com
+pet.ibomma.bond
+www.xvideos.com
+drive.google.com
+pl.escort.club
+yaoilib.me
+znanija.com
+free.facebook.com
+wl1.peachavocado.com
+www.shutterstock.com
+www.eporner.com
+www2.hm.com
+newtoki328.com
+woman.excite.co.jp
+www.xvideos5.com
+br.search.yahoo.com
+spankbang.com
+www.gismeteo.ru
+www.uol.com.br
+fetlife.com
+serviziweb2.inps.it
+aternos.org
+wiadomosci.wp.pl
+www.speedtest.net
+gamewith.jp
+www.tuazar.com
+it.pornhub.com
+web.facebook.com
+www.erome.com
+www.screener.in
+client--gws-wiz-serp22-v1ddlcxru01-ioin.a-0.xyz
+services.gst.gov.in
+snapinsta.app
+www.xvideos2.com
+tw.news.yahoo.com
+one.google.com
+classroom.google.com
+happymod.com
+satta-king-fast.com
+remaker.ai
+www.ecosia.org
+video.hnext.jp
+pt.m.wikipedia.org
+open.spotify.com
+translate.google.com
+nhentai.net
+ww1.goojara.to
+m.site777.jp
+www.upwork.com
+www.terra.com.br
+www.sanook.com
+wapbold.net
+www.xnxx.com
+www.amazon.co.uk
+www.njuskalo.hr
+www.nairaland.com
+s.tabelog.com
+www.tbshop-ma.net
+m.cricbuzz.com
+www.amazon.ca
+www.expressen.se
+www.kleinanzeigen.de
+xxxhindi.to
+heylink.me
+www.dlsite.com
+www.hibiny.ru
+www.corriere.it
+www.nytimes.com
+xnxxsex.net
+www.ups.com
+rx.sp.mbga.jp
+brainly.com.br
+www.hepsiburada.com
+www.tripadvisor.com
+sportowefakty.wp.pl
+www.indianbfvideos.com
+82xnxx.com
+m.daum.net
+manhuascan.io
+www.mercadolibre.com.mx
+spankbang.party
+mangatoto.com
+en.wikipedia.org
+www.amazon.it
+persona.patria.org.ve
+www.tamildhool.net
+zinmanga.com
+www.otto.de
+91club06.com
+www.pornpics.com
+suumo.jp
+www.boatrace.jp
+www.infobae.com
+m.inven.co.kr
+quizlet.com
+turbo.az
+www.y2mate.com
+www.toppr.com
+studio.youtube.com
+m.avito.ru
+blog.livedoor.jp
+m.fabguys.com
+www.microsoft.com
+www.bol.com
+www.nitrotype.com
+www.temu.com
+goal.co
+www.khabaronline.ir
+91club.club
+allegro.pl
+funnyjunk.com
+auctions.yahoo.co.jp
+pt.pornhub.com
+lermanga.org
+rajshaladarpan.nic.in
+wto.to
+m.scan-manga.com
+www.canva.com
+play.contents.plus-msg.auone.jp
+ja.m.wikipedia.org
+www.carsensor.net
+www.nettruyenee.com
+zoom.us
+www.sarkariresult.com
+page.auctions.yahoo.co.jp
+shopee.co.id
+www.nesine.com
+www.wnacg.com
+www.dafont.com
+www.theguardian.com
+www.sinsay.com
+a24.app.gree-pf.net
+roboguru.ruangguru.com
+www.studocu.com
+www.timesbull.com
+yatv.pro
+nouen.sp.mbga.jp
+cardgames.io
+sp.pf.mbga.jp
+passbook.epfindia.gov.in
+pg.access-33.com
+wonporn.com
+yandex.com
+noutore.enjoy.point.auone.jp
+www.primevideo.com
+articulo.mercadolibre.com.mx
+www.pornhub.com
+www.rightmove.co.uk
+es.scribd.com
+jp.pornhub.com
+www.repubblica.it
+id.m.wikipedia.org
+m.skelbiu.lt
+www.marktplaats.nl
+es.m.wikipedia.org
+search.yahoo.com
+jp.spankbang.com

From bb8337736e2a5c364574335efad63b8e38da9a62 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Fri, 24 May 2024 10:13:01 -0700
Subject: [PATCH 02/23] seeing results

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8a265b2a..6f81568f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -40,4 +40,4 @@ jobs:
       - name: Large-Scale Integration Test
         run: |
           python --version
-          cat largescaleintegration/top-1k.csv | ./zdns A --iterative | grep "TIMEOUT" | wc -l
+          cat largescaleintegration/top-1k.csv | ./zdns A --iterative 

From 75e5fdcb4768ac742184bbb5b6dddb4ade0b3852 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Fri, 24 May 2024 15:08:47 -0700
Subject: [PATCH 03/23] able to request X domain to Y IP

---
 largescaleintegration/large_scan_validator.py |   0
 largescaleintegration/test_validators.py      |   0
 largescaleintegration/validators.py           | 108 ++++++++++++++++++
 3 files changed, 108 insertions(+)
 create mode 100644 largescaleintegration/large_scan_validator.py
 create mode 100644 largescaleintegration/test_validators.py
 create mode 100644 largescaleintegration/validators.py

diff --git a/largescaleintegration/large_scan_validator.py b/largescaleintegration/large_scan_validator.py
new file mode 100644
index 00000000..e69de29b
diff --git a/largescaleintegration/test_validators.py b/largescaleintegration/test_validators.py
new file mode 100644
index 00000000..e69de29b
diff --git a/largescaleintegration/validators.py b/largescaleintegration/validators.py
new file mode 100644
index 00000000..7675cb1a
--- /dev/null
+++ b/largescaleintegration/validators.py
@@ -0,0 +1,108 @@
+from typing import List
+
+import requests
+import socket
+import concurrent.futures
+import asyncio
+from pyppeteer import launch
+
+
+# async def can_visit_with_puppeteer(domain: str, ip_address: str) -> bool:
+#     browser = await launch(headless=True, ignoreHTTPSErrors=True)
+#     page = await browser.newPage()
+#
+#     try:
+#         await page.setExtraHTTPHeaders({'Host': domain,
+#                                         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15'})
+#         response = await page.goto(f"https://{ip_address}", {'waitUntil': 'networkidle0'})
+#
+#         # Check if the navigation was successful (status code 200)
+#         if response.ok:
+#             # Take a screenshot of the page
+#             await page.screenshot({'path': '~/zdns/zdns-working/largescaleintegration/screenshot.png'})
+#             return True
+#         else:
+#             return False
+#
+#     except Exception as e:
+#         print(f"An error occurred with domain {domain} at IP {ip_address}: {e}")
+#         return False
+#     finally:
+#         await browser.close()
+
+
+def can_request_successfully(domain: str, ip: str) -> bool:
+    user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15"
+    headers = {"Host": domain, "User-Agent": user_agent}
+    try:
+        response = requests.get("https://" + ip, headers=headers, verify=False, timeout=5)
+        if response.status_code < 400:
+            print(f"Failed to request {domain} with requests with status code {response.status_code}")
+            return True
+        return False
+    except requests.exceptions.RequestException as e:
+        print(f"Failed to request {domain} with requests with error {e}")
+        return False
+
+
+def can_automatically_visit(domain: str, ip: str):
+    if can_request_successfully(domain, ip):
+        return (domain, ip, 0)
+    # if domain == "arca.live":
+    #     ip = "1.1.1.1"
+    # if await can_visit_with_puppeteer(domain, ip):
+    #     return 1
+    return (domain, ip, -1)
+
+def get_ip_address(domain: str):
+    try:
+        ip = socket.gethostbyname(domain)
+        print(f"Resolved {domain} to {ip}")
+        return (domain, ip)
+    except socket.gaierror:
+        return (domain, "")
+
+
+# Returns a list of tuples, where each tuple contains the domain and the IP address ZDNS returned. A domain could have
+# multiple IP addresses, so there could be multiple tuples for a single domain.
+def get_zdns_results(domains: List[str]) -> List[tuple[str, str]]:
+    return []
+
+
+def main():
+    # read in domains file
+    with open("top-1k.csv", "r") as f:
+        domains = f.read().splitlines()
+
+
+    # Testing, only use the first 10 domains
+    domains = domains[0:100]
+    domain_ip_pairs = []
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        results = [executor.submit(get_ip_address, domain) for domain in domains]
+
+        for future in concurrent.futures.as_completed(results):
+            domain, ip = future.result()
+            domain_ip_pairs.append((domain, ip))
+
+    successes_requests = []
+    successes_puppeteer = []
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        results = [executor.submit(can_automatically_visit, domain, ip) for domain, ip in domain_ip_pairs]
+
+        for future in concurrent.futures.as_completed(results):
+            domain, ip, result = future.result()
+            if result == 0:
+                successes_requests.append(domain)
+            elif result == 1:
+                successes_puppeteer.append(domain)
+            elif result == -1:
+                print(f"Failed to visit {domain} with either requests or puppeteer")
+
+    print(f"Successfully requested {len(successes_requests)} out of {len(domains)} domains with requests")
+    print(f"Successfully visited {len(successes_puppeteer)} out of {len(domains)} domains with puppeteer")
+    print("Requests successes:", successes_requests)
+    print("Puppeteer successes:", successes_puppeteer)
+    print("Failures:", set(domains) - set(successes_requests) - set(successes_puppeteer))
+
+main()
\ No newline at end of file

From ab3c87c7f1712daf1f345f347a10f331cb54a7e3 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Fri, 24 May 2024 16:25:39 -0700
Subject: [PATCH 04/23] working a and alookup tests

---
 largescaleintegration/validators.py | 119 ++++++++++++++++++++++++----
 1 file changed, 103 insertions(+), 16 deletions(-)

diff --git a/largescaleintegration/validators.py b/largescaleintegration/validators.py
index 7675cb1a..71830530 100644
--- a/largescaleintegration/validators.py
+++ b/largescaleintegration/validators.py
@@ -1,3 +1,4 @@
+import os
 from typing import List
 
 import requests
@@ -5,6 +6,12 @@
 import concurrent.futures
 import asyncio
 from pyppeteer import launch
+import select
+import subprocess
+import unittest
+import json
+
+ZDNS_EXECUTABLE = "./zdns"
 
 
 # async def can_visit_with_puppeteer(domain: str, ip_address: str) -> bool:
@@ -37,8 +44,8 @@ def can_request_successfully(domain: str, ip: str) -> bool:
     try:
         response = requests.get("https://" + ip, headers=headers, verify=False, timeout=5)
         if response.status_code < 400:
-            print(f"Failed to request {domain} with requests with status code {response.status_code}")
             return True
+        print(f"Failed to request {domain} with requests with status code {response.status_code}")
         return False
     except requests.exceptions.RequestException as e:
         print(f"Failed to request {domain} with requests with error {e}")
@@ -54,6 +61,7 @@ def can_automatically_visit(domain: str, ip: str):
     #     return 1
     return (domain, ip, -1)
 
+
 def get_ip_address(domain: str):
     try:
         ip = socket.gethostbyname(domain)
@@ -63,20 +71,40 @@ def get_ip_address(domain: str):
         return (domain, "")
 
 
-# Returns a list of tuples, where each tuple contains the domain and the IP address ZDNS returned. A domain could have
-# multiple IP addresses, so there could be multiple tuples for a single domain.
-def get_zdns_results(domains: List[str]) -> List[tuple[str, str]]:
-    return []
+def run_zdns(input_domains, flags, executable=ZDNS_EXECUTABLE):
+    # pipe the input domains into a call to zdns
+    # return the output of zdns
+    # Convert the list of domains to a single string, with each domain on a new line
+    input_data = "\n".join(input_domains)
+
+    # Start the zdns subprocess
+    print(flags)
+    process = subprocess.Popen([executable] + flags, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+
+    # Send the input data to the subprocess and get the output
+    process.stdin.write(input_data.encode())
+    process.stdin.close()
+
+    output = []
+
+    while True:
+        # Use select to wait for data to become available
+        reads = [process.stdout.fileno(), process.stderr.fileno()]
+        ret = select.select(reads, [], [])
 
+        for fd in ret[0]:
+            if fd == process.stdout.fileno():
+                output.append(process.stdout.readline().decode())
+            if fd == process.stderr.fileno():
+                print("stderr: " + process.stderr.readline().decode())
+        if process.poll() is not None:
+            break
 
-def main():
-    # read in domains file
-    with open("top-1k.csv", "r") as f:
-        domains = f.read().splitlines()
+    return output
 
 
-    # Testing, only use the first 10 domains
-    domains = domains[0:100]
+def get_requestable_domains(domains: List[str]) -> List[str]:
     domain_ip_pairs = []
     with concurrent.futures.ThreadPoolExecutor() as executor:
         results = [executor.submit(get_ip_address, domain) for domain in domains]
@@ -94,15 +122,74 @@ def main():
             domain, ip, result = future.result()
             if result == 0:
                 successes_requests.append(domain)
-            elif result == 1:
-                successes_puppeteer.append(domain)
+            # elif result == 1:
+            #     successes_puppeteer.append(domain)
             elif result == -1:
                 print(f"Failed to visit {domain} with either requests or puppeteer")
 
     print(f"Successfully requested {len(successes_requests)} out of {len(domains)} domains with requests")
-    print(f"Successfully visited {len(successes_puppeteer)} out of {len(domains)} domains with puppeteer")
+    # print(f"Successfully visited {len(successes_puppeteer)} out of {len(domains)} domains with puppeteer")
     print("Requests successes:", successes_requests)
-    print("Puppeteer successes:", successes_puppeteer)
+    # print("Puppeteer successes:", successes_puppeteer)
     print("Failures:", set(domains) - set(successes_requests) - set(successes_puppeteer))
+    return successes_requests
 
-main()
\ No newline at end of file
+
+# Returns a list of tuples, where each tuple contains the domain and the IP address ZDNS returned. A domain could have
+# multiple IP addresses, so there could be multiple tuples for a single domain.
+def get_zdns_results_a(domains: List[str]) -> List[tuple[str, str]]:
+    zdns_output = run_zdns(domains, ["A", "--iterative"])
+    domain_ip_pairs = []
+    for line in zdns_output:
+        if "answer" in line and "TIMEOUT" not in line and "CNAME" not in line:
+            domain = line.split('"name":"')[1].split('"')[0]
+            ip = line.split('"answer":"')[1].split('"')[0]
+            domain_ip_pairs.append((domain, ip))
+        else:
+            print(f"Failed to resolve domain: {line}")
+    return domain_ip_pairs
+
+
+def get_zdns_results_a_lookup(domains: List[str]) -> List[tuple[str, str]]:
+    zdns_output = run_zdns(domains, ["ALOOKUP", "--iterative"])
+    domain_ip_pairs = []
+    for line in zdns_output:
+        if "ipv4_addresses" in line and "TIMEOUT" not in line:
+            # use jq to parse the JSON output, to extract data.ipv4_addresses
+            line_data = json.loads(line)
+            data = line_data.get('data')
+            ipv4_addresses = data.get('ipv4_addresses') if data else None
+            if ipv4_addresses:
+                for ip in ipv4_addresses:
+                    domain = line_data.get('name')
+                    domain_ip_pairs.append((domain, ip))
+        else:
+            print(f"Failed to resolve domain: {line}")
+    return domain_ip_pairs
+
+
+class TestZDNS(unittest.TestCase):
+    def setUp(self):
+        with open("top-1k.csv", "r") as f:
+            domains = f.read().splitlines()
+
+        domains = domains[0:100]
+        # These are those domains which we can successfully request using the requests library directed at an IP address
+        # This excludes domains that use some form of DDoS mitigation, such as Cloudflare, which have more sophisticated
+        # bot detection
+        self.known_reachable_domains = get_requestable_domains(domains)
+
+    def test_zdns_a(self):
+        zdns = get_zdns_results_a(self.known_reachable_domains)
+        for domain, ip in zdns:
+            self.assertTrue(can_request_successfully(domain, ip),
+                            "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
+
+    def test_zdns_a_lookup(self):
+        zdns = get_zdns_results_a_lookup(self.known_reachable_domains)
+        for domain, ip in zdns:
+            self.assertTrue(can_request_successfully(domain, ip), "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
+
+
+if __name__ == '__main__':
+    unittest.main()

From 2a789cee3a711cfdf0ecadbdb06552806a986dcc Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 09:30:48 -0700
Subject: [PATCH 05/23] working validators

---
 largescaleintegration/large_scan_validator.py | 65 +++++++++++++++++++
 largescaleintegration/test_validators.py      | 34 ++++++++++
 2 files changed, 99 insertions(+)

diff --git a/largescaleintegration/large_scan_validator.py b/largescaleintegration/large_scan_validator.py
index e69de29b..737d9508 100644
--- a/largescaleintegration/large_scan_validator.py
+++ b/largescaleintegration/large_scan_validator.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+import select
+import subprocess
+
+ZDNS_EXECUTABLE = "./zdns"
+LIST_OF_DOMAINS_FILE = "./top-1k.csv"
+
+# Validate a line of output from zdns
+# Example output line:
+#{"data":{"answers":[{"answer":"104.16.25.14","class":"IN","name":"www.patreon.com","ttl":300,"type":"A"},{"answer":"104.16.24.14","class":"IN","name":"www.patreon.com","ttl":300,"type":"A"}],"protocol":"udp","resolver":"108.162.193.149:53"},"name":"www.patreon.com","status":"NOERROR","timestamp":"2024-05-24T16:56:16Z"}
+#{"data":{"answers":[{"answer":"ds-ats.member.g02.yahoodns.net.","class":"IN","name":"login.yahoo.com","ttl":300,"type":"CNAME"}],"protocol":"udp","resolver":"68.180.131.16:53"},"name":"login.yahoo.com","status":"NOERROR","timestamp":"2024-05-24T16:56:16Z"}
+#{"data":{"protocol":"udp","resolver":"192.203.230.10:53"},"name":"discord.com","status":"TIMEOUT","timestamp":"2024-05-24T16:56:24Z"}
+
+class ZDNSScanTest:
+    timeouts = 0
+    correct_answers = 0
+    incorrect_answers = 0
+    def __init__(self):
+
+def validate_zdns_output_line(output_line):
+    # Parse the JSON output line
+    # Need to determine if the line encountered a timeout, then
+
+# Run zdns on a list of domains
+def run_zdns(input_domains, flags, executable=ZDNS_EXECUTABLE):
+    # pipe the input domains into a call to zdns
+    # return the output of zdns
+    # Convert the list of domains to a single string, with each domain on a new line
+    input_data = "\n".join(input_domains)
+
+    # Start the zdns subprocess
+    print(flags)
+    process = subprocess.Popen([executable] + flags, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    # Send the input data to the subprocess and get the output
+    process.stdin.write(input_data.encode())
+    process.stdin.close()
+
+    while True:
+        # Use select to wait for data to become available
+        reads = [process.stdout.fileno(), process.stderr.fileno()]
+        ret = select.select(reads, [], [])
+
+        for fd in ret[0]:
+            if fd == process.stdout.fileno():
+                validate_zdns_output_line(process.stdout.readline().decode())
+            if fd == process.stderr.fileno():
+                print("stderr: " + process.stderr.readline().decode())
+        if process.poll() is not None:
+            break
+    # # Convert the output data from bytes to string
+    # stdout_data = process.stdout.readline().decode()
+    # stderr_data = process.stderr.readline().decode()
+
+    # Return the output data
+    # return stdout_data, stderr_data
+
+if __name__ == '__main__':
+    # Read the list of domains from the file
+    with open(LIST_OF_DOMAINS_FILE, 'r') as f:
+        domains = f.read().splitlines()
+
+    # Run zdns on the list of domains
+    print(domains[0])
+    run_zdns(domains[9:10], ["A", "--iterative"])
diff --git a/largescaleintegration/test_validators.py b/largescaleintegration/test_validators.py
index e69de29b..4e7dbd8d 100644
--- a/largescaleintegration/test_validators.py
+++ b/largescaleintegration/test_validators.py
@@ -0,0 +1,34 @@
+import unittest
+from validators import verify_A_record
+
+
+
+class TestVerifyARecord(unittest.TestCase):
+    def test_known_good_domain_ip_pair(self):
+        domain = "one.one.one.one"  # Cloudflare's domain
+        ip = "1.1.1.1"  # Hosted by cloudflare
+        self.assertTrue(verify_A_record(domain, ip))
+
+    def test_known_bad_domain_ip_pair(self):
+        domain = "one.one.one.one"
+        ip = "171.67.70.3"  # An IP within Stanford's IP range, not Cloudflare's
+        self.assertFalse(verify_A_record(domain, ip))
+
+    CLOUDFLARE_HOSTING_IP = "104.21.96.65"
+    AWS_HOSTING_IP = "75.2.60.5"
+    cloudflare_domain = "prstephens.com"
+    aws_hosted_domain = "clifbar.com"
+    def test_good_domain_ip_pair_with_sni(self):
+        # Since this IP address is a hosting endpoint for Cloudflare, it requires SNI to access cloudflare.com
+        self.assertTrue(verify_A_record(self.cloudflare_domain, self.CLOUDFLARE_HOSTING_IP))
+        self.assertTrue(verify_A_record(self.aws_hosted_domain, self.AWS_HOSTING_IP))
+
+    def test_bad_domain_ip_pair_with_sni(self):
+        self.assertFalse(verify_A_record(self.cloudflare_domain, self.AWS_HOSTING_IP))
+        self.assertFalse(verify_A_record(self.aws_hosted_domain, self.CLOUDFLARE_HOSTING_IP))
+
+
+
+
+if __name__ == '__main__':
+    unittest.main()

From dec7f036eba678533d62e979568f81eb8977ecd1 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 13:59:31 -0700
Subject: [PATCH 06/23] cleaning up unneeded files and added some logs

---
 largescaleintegration/test_validators.py | 34 ---------------------
 largescaleintegration/validators.py      | 39 ++++--------------------
 2 files changed, 6 insertions(+), 67 deletions(-)
 delete mode 100644 largescaleintegration/test_validators.py

diff --git a/largescaleintegration/test_validators.py b/largescaleintegration/test_validators.py
deleted file mode 100644
index 4e7dbd8d..00000000
--- a/largescaleintegration/test_validators.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import unittest
-from validators import verify_A_record
-
-
-
-class TestVerifyARecord(unittest.TestCase):
-    def test_known_good_domain_ip_pair(self):
-        domain = "one.one.one.one"  # Cloudflare's domain
-        ip = "1.1.1.1"  # Hosted by cloudflare
-        self.assertTrue(verify_A_record(domain, ip))
-
-    def test_known_bad_domain_ip_pair(self):
-        domain = "one.one.one.one"
-        ip = "171.67.70.3"  # An IP within Stanford's IP range, not Cloudflare's
-        self.assertFalse(verify_A_record(domain, ip))
-
-    CLOUDFLARE_HOSTING_IP = "104.21.96.65"
-    AWS_HOSTING_IP = "75.2.60.5"
-    cloudflare_domain = "prstephens.com"
-    aws_hosted_domain = "clifbar.com"
-    def test_good_domain_ip_pair_with_sni(self):
-        # Since this IP address is a hosting endpoint for Cloudflare, it requires SNI to access cloudflare.com
-        self.assertTrue(verify_A_record(self.cloudflare_domain, self.CLOUDFLARE_HOSTING_IP))
-        self.assertTrue(verify_A_record(self.aws_hosted_domain, self.AWS_HOSTING_IP))
-
-    def test_bad_domain_ip_pair_with_sni(self):
-        self.assertFalse(verify_A_record(self.cloudflare_domain, self.AWS_HOSTING_IP))
-        self.assertFalse(verify_A_record(self.aws_hosted_domain, self.CLOUDFLARE_HOSTING_IP))
-
-
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/largescaleintegration/validators.py b/largescaleintegration/validators.py
index 71830530..18ee9d74 100644
--- a/largescaleintegration/validators.py
+++ b/largescaleintegration/validators.py
@@ -13,31 +13,6 @@
 
 ZDNS_EXECUTABLE = "./zdns"
 
-
-# async def can_visit_with_puppeteer(domain: str, ip_address: str) -> bool:
-#     browser = await launch(headless=True, ignoreHTTPSErrors=True)
-#     page = await browser.newPage()
-#
-#     try:
-#         await page.setExtraHTTPHeaders({'Host': domain,
-#                                         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15'})
-#         response = await page.goto(f"https://{ip_address}", {'waitUntil': 'networkidle0'})
-#
-#         # Check if the navigation was successful (status code 200)
-#         if response.ok:
-#             # Take a screenshot of the page
-#             await page.screenshot({'path': '~/zdns/zdns-working/largescaleintegration/screenshot.png'})
-#             return True
-#         else:
-#             return False
-#
-#     except Exception as e:
-#         print(f"An error occurred with domain {domain} at IP {ip_address}: {e}")
-#         return False
-#     finally:
-#         await browser.close()
-
-
 def can_request_successfully(domain: str, ip: str) -> bool:
     user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15"
     headers = {"Host": domain, "User-Agent": user_agent}
@@ -55,10 +30,6 @@ def can_request_successfully(domain: str, ip: str) -> bool:
 def can_automatically_visit(domain: str, ip: str):
     if can_request_successfully(domain, ip):
         return (domain, ip, 0)
-    # if domain == "arca.live":
-    #     ip = "1.1.1.1"
-    # if await can_visit_with_puppeteer(domain, ip):
-    #     return 1
     return (domain, ip, -1)
 
 
@@ -114,7 +85,6 @@ def get_requestable_domains(domains: List[str]) -> List[str]:
             domain_ip_pairs.append((domain, ip))
 
     successes_requests = []
-    successes_puppeteer = []
     with concurrent.futures.ThreadPoolExecutor() as executor:
         results = [executor.submit(can_automatically_visit, domain, ip) for domain, ip in domain_ip_pairs]
 
@@ -131,7 +101,7 @@ def get_requestable_domains(domains: List[str]) -> List[str]:
     # print(f"Successfully visited {len(successes_puppeteer)} out of {len(domains)} domains with puppeteer")
     print("Requests successes:", successes_requests)
     # print("Puppeteer successes:", successes_puppeteer)
-    print("Failures:", set(domains) - set(successes_requests) - set(successes_puppeteer))
+    print("Failures:", set(domains) - set(successes_requests))
     return successes_requests
 
 
@@ -169,7 +139,8 @@ def get_zdns_results_a_lookup(domains: List[str]) -> List[tuple[str, str]]:
 
 
 class TestZDNS(unittest.TestCase):
-    def setUp(self):
+    @classmethod
+    def setUpClass(cls):
         with open("top-1k.csv", "r") as f:
             domains = f.read().splitlines()
 
@@ -177,16 +148,18 @@ def setUp(self):
         # These are those domains which we can successfully request using the requests library directed at an IP address
         # This excludes domains that use some form of DDoS mitigation, such as Cloudflare, which have more sophisticated
         # bot detection
-        self.known_reachable_domains = get_requestable_domains(domains)
+        cls.known_reachable_domains = get_requestable_domains(domains)
 
     def test_zdns_a(self):
         zdns = get_zdns_results_a(self.known_reachable_domains)
+        print(f"ZDNS resolved {len(zdns)} domains to IP addresses from request-able domains")
         for domain, ip in zdns:
             self.assertTrue(can_request_successfully(domain, ip),
                             "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
 
     def test_zdns_a_lookup(self):
         zdns = get_zdns_results_a_lookup(self.known_reachable_domains)
+        print(f"ZDNS resolved {len(zdns)} domains to IP addresses from request-able domains")
         for domain, ip in zdns:
             self.assertTrue(can_request_successfully(domain, ip), "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
 

From f48992e5ff6fb2ea91ab860eaa77c98dad75f792 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:09:31 -0700
Subject: [PATCH 07/23] cleanup and add as github action

---
 .github/workflows/ci.yml                      | 17 ++++-
 largescaleintegration/large_scan_validator.py | 65 -------------------
 .../integration_tests.py                      |  2 +-
 {largescaleintegration => testing}/top-1k.csv |  0
 .../top_1k_domains_integration_tests.py       |  2 +-
 5 files changed, 17 insertions(+), 69 deletions(-)
 delete mode 100644 largescaleintegration/large_scan_validator.py
 rename integration_tests.py => testing/integration_tests.py (99%)
 rename {largescaleintegration => testing}/top-1k.csv (100%)
 rename largescaleintegration/validators.py => testing/top_1k_domains_integration_tests.py (99%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f81568f..32b6e62f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -36,8 +36,21 @@ jobs:
           sudo rm /etc/resolv.conf
           sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
           python --version
-          ./integration_tests.py
+          ./testing/integration_tests.py
+  build-and-test-top-1k:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.20'
+      - name: Build
+        run: |
+          go version
+          make
       - name: Large-Scale Integration Test
         run: |
           python --version
-          cat largescaleintegration/top-1k.csv | ./zdns A --iterative 
+          ./testing/large_scale_integration_test.py
diff --git a/largescaleintegration/large_scan_validator.py b/largescaleintegration/large_scan_validator.py
deleted file mode 100644
index 737d9508..00000000
--- a/largescaleintegration/large_scan_validator.py
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-import select
-import subprocess
-
-ZDNS_EXECUTABLE = "./zdns"
-LIST_OF_DOMAINS_FILE = "./top-1k.csv"
-
-# Validate a line of output from zdns
-# Example output line:
-#{"data":{"answers":[{"answer":"104.16.25.14","class":"IN","name":"www.patreon.com","ttl":300,"type":"A"},{"answer":"104.16.24.14","class":"IN","name":"www.patreon.com","ttl":300,"type":"A"}],"protocol":"udp","resolver":"108.162.193.149:53"},"name":"www.patreon.com","status":"NOERROR","timestamp":"2024-05-24T16:56:16Z"}
-#{"data":{"answers":[{"answer":"ds-ats.member.g02.yahoodns.net.","class":"IN","name":"login.yahoo.com","ttl":300,"type":"CNAME"}],"protocol":"udp","resolver":"68.180.131.16:53"},"name":"login.yahoo.com","status":"NOERROR","timestamp":"2024-05-24T16:56:16Z"}
-#{"data":{"protocol":"udp","resolver":"192.203.230.10:53"},"name":"discord.com","status":"TIMEOUT","timestamp":"2024-05-24T16:56:24Z"}
-
-class ZDNSScanTest:
-    timeouts = 0
-    correct_answers = 0
-    incorrect_answers = 0
-    def __init__(self):
-
-def validate_zdns_output_line(output_line):
-    # Parse the JSON output line
-    # Need to determine if the line encountered a timeout, then
-
-# Run zdns on a list of domains
-def run_zdns(input_domains, flags, executable=ZDNS_EXECUTABLE):
-    # pipe the input domains into a call to zdns
-    # return the output of zdns
-    # Convert the list of domains to a single string, with each domain on a new line
-    input_data = "\n".join(input_domains)
-
-    # Start the zdns subprocess
-    print(flags)
-    process = subprocess.Popen([executable] + flags, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-    # Send the input data to the subprocess and get the output
-    process.stdin.write(input_data.encode())
-    process.stdin.close()
-
-    while True:
-        # Use select to wait for data to become available
-        reads = [process.stdout.fileno(), process.stderr.fileno()]
-        ret = select.select(reads, [], [])
-
-        for fd in ret[0]:
-            if fd == process.stdout.fileno():
-                validate_zdns_output_line(process.stdout.readline().decode())
-            if fd == process.stderr.fileno():
-                print("stderr: " + process.stderr.readline().decode())
-        if process.poll() is not None:
-            break
-    # # Convert the output data from bytes to string
-    # stdout_data = process.stdout.readline().decode()
-    # stderr_data = process.stderr.readline().decode()
-
-    # Return the output data
-    # return stdout_data, stderr_data
-
-if __name__ == '__main__':
-    # Read the list of domains from the file
-    with open(LIST_OF_DOMAINS_FILE, 'r') as f:
-        domains = f.read().splitlines()
-
-    # Run zdns on the list of domains
-    print(domains[0])
-    run_zdns(domains[9:10], ["A", "--iterative"])
diff --git a/integration_tests.py b/testing/integration_tests.py
similarity index 99%
rename from integration_tests.py
rename to testing/integration_tests.py
index 61bed8ae..a4a04094 100755
--- a/integration_tests.py
+++ b/testing/integration_tests.py
@@ -39,7 +39,7 @@ def dictSort(d):
 class Tests(unittest.TestCase):
 
     maxDiff = None
-    ZDNS_EXECUTABLE = "./zdns" 
+    ZDNS_EXECUTABLE = "../zdns"
 
     def run_zdns_check_failure(self, flags, name, expected_err, executable=ZDNS_EXECUTABLE):
         flags = flags + " --threads=10"
diff --git a/largescaleintegration/top-1k.csv b/testing/top-1k.csv
similarity index 100%
rename from largescaleintegration/top-1k.csv
rename to testing/top-1k.csv
diff --git a/largescaleintegration/validators.py b/testing/top_1k_domains_integration_tests.py
similarity index 99%
rename from largescaleintegration/validators.py
rename to testing/top_1k_domains_integration_tests.py
index 18ee9d74..ad0c7c6e 100644
--- a/largescaleintegration/validators.py
+++ b/testing/top_1k_domains_integration_tests.py
@@ -11,7 +11,7 @@
 import unittest
 import json
 
-ZDNS_EXECUTABLE = "./zdns"
+ZDNS_EXECUTABLE = "../zdns"
 
 def can_request_successfully(domain: str, ip: str) -> bool:
     user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15"

From 202d76691c9733a6c88dfa43abd1d2b04995977f Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:41:25 -0700
Subject: [PATCH 08/23] updated zdns exe path

---
 testing/integration_tests.py                | 2 +-
 testing/top_1k_domains_integration_tests.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/testing/integration_tests.py b/testing/integration_tests.py
index a4a04094..b366bc23 100755
--- a/testing/integration_tests.py
+++ b/testing/integration_tests.py
@@ -39,7 +39,7 @@ def dictSort(d):
 class Tests(unittest.TestCase):
 
     maxDiff = None
-    ZDNS_EXECUTABLE = "../zdns"
+    ZDNS_EXECUTABLE = "./zdns"
 
     def run_zdns_check_failure(self, flags, name, expected_err, executable=ZDNS_EXECUTABLE):
         flags = flags + " --threads=10"
diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_1k_domains_integration_tests.py
index ad0c7c6e..18ee9d74 100644
--- a/testing/top_1k_domains_integration_tests.py
+++ b/testing/top_1k_domains_integration_tests.py
@@ -11,7 +11,7 @@
 import unittest
 import json
 
-ZDNS_EXECUTABLE = "../zdns"
+ZDNS_EXECUTABLE = "./zdns"
 
 def can_request_successfully(domain: str, ip: str) -> bool:
     user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15"

From f7066127b11b43495c27aca50840c8b459adf0c7 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:42:53 -0700
Subject: [PATCH 09/23] updated test name in ci.yml

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 32b6e62f..da0fe90a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -53,4 +53,4 @@ jobs:
       - name: Large-Scale Integration Test
         run: |
           python --version
-          ./testing/large_scale_integration_test.py
+          ./testing/top_1k_domains_integration_tests.py

From 48a16c64a563f4b10fb85beb46ab813cff537c80 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:45:50 -0700
Subject: [PATCH 10/23] made python test a script

---
 testing/top_1k_domains_integration_tests.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_1k_domains_integration_tests.py
index 18ee9d74..0d608384 100644
--- a/testing/top_1k_domains_integration_tests.py
+++ b/testing/top_1k_domains_integration_tests.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os
 from typing import List
 

From 2eeeae9e8b4989c93223fda3a7854e5ea07c0f5b Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:48:03 -0700
Subject: [PATCH 11/23] updated script permissions

---
 testing/top_1k_domains_integration_tests.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 testing/top_1k_domains_integration_tests.py

diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_1k_domains_integration_tests.py
old mode 100644
new mode 100755

From 1855325df73fc079dbf93af01b2be5f9c375c85e Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:49:32 -0700
Subject: [PATCH 12/23] removed dead import

---
 testing/top_1k_domains_integration_tests.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_1k_domains_integration_tests.py
index 0d608384..9d0fd1e1 100755
--- a/testing/top_1k_domains_integration_tests.py
+++ b/testing/top_1k_domains_integration_tests.py
@@ -1,13 +1,10 @@
 #!/usr/bin/env python3
 
-import os
 from typing import List
 
 import requests
 import socket
 import concurrent.futures
-import asyncio
-from pyppeteer import launch
 import select
 import subprocess
 import unittest

From f5cc9e3dcbfac0d9cb8f8fc1d80873a01b9141f8 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:51:09 -0700
Subject: [PATCH 13/23] updated file name

---
 testing/top_1k_domains_integration_tests.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_1k_domains_integration_tests.py
index 9d0fd1e1..351779cd 100755
--- a/testing/top_1k_domains_integration_tests.py
+++ b/testing/top_1k_domains_integration_tests.py
@@ -11,6 +11,7 @@
 import json
 
 ZDNS_EXECUTABLE = "./zdns"
+TOP_1K_DOMAINS_FILE = "./testing/top-1k.csv"
 
 def can_request_successfully(domain: str, ip: str) -> bool:
     user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15"
@@ -140,7 +141,7 @@ def get_zdns_results_a_lookup(domains: List[str]) -> List[tuple[str, str]]:
 class TestZDNS(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        with open("top-1k.csv", "r") as f:
+        with open(TOP_1K_DOMAINS_FILE, "r") as f:
             domains = f.read().splitlines()
 
         domains = domains[0:100]

From e0ca48b08e1cbe994696596d6cac2c5ea42346a7 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 14:53:39 -0700
Subject: [PATCH 14/23] made error msg less scary

---
 testing/top_1k_domains_integration_tests.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_1k_domains_integration_tests.py
index 351779cd..ce324673 100755
--- a/testing/top_1k_domains_integration_tests.py
+++ b/testing/top_1k_domains_integration_tests.py
@@ -116,7 +116,7 @@ def get_zdns_results_a(domains: List[str]) -> List[tuple[str, str]]:
             ip = line.split('"answer":"')[1].split('"')[0]
             domain_ip_pairs.append((domain, ip))
         else:
-            print(f"Failed to resolve domain: {line}")
+            print(f"Failed to parse ZDNS output: {line}")
     return domain_ip_pairs
 
 
@@ -134,7 +134,7 @@ def get_zdns_results_a_lookup(domains: List[str]) -> List[tuple[str, str]]:
                     domain = line_data.get('name')
                     domain_ip_pairs.append((domain, ip))
         else:
-            print(f"Failed to resolve domain: {line}")
+            print(f"Failed to parse ZDNS output: {line}")
     return domain_ip_pairs
 
 

From fb07c87393057cad53578ac97f5489a7cd064775 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:14:23 -0700
Subject: [PATCH 15/23] only run CI on commits to main branch and all PR's,
 otherwise we're doubling the effort on all PRs

---
 .github/workflows/ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index da0fe90a..61bf8b1f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,5 +1,9 @@
 name: CI
-on: [ push, pull_request ]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 jobs:
   check-license:
     runs-on: ubuntu-latest

From b7005b1ff65f821faba39b727e9ad0df2a83e9df Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:16:02 -0700
Subject: [PATCH 16/23] remove limit to top 100, run against all 1k domains

---
 testing/top_1k_domains_integration_tests.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_1k_domains_integration_tests.py
index ce324673..6f0fb08f 100755
--- a/testing/top_1k_domains_integration_tests.py
+++ b/testing/top_1k_domains_integration_tests.py
@@ -143,8 +143,6 @@ class TestZDNS(unittest.TestCase):
     def setUpClass(cls):
         with open(TOP_1K_DOMAINS_FILE, "r") as f:
             domains = f.read().splitlines()
-
-        domains = domains[0:100]
         # These are those domains which we can successfully request using the requests library directed at an IP address
         # This excludes domains that use some form of DDoS mitigation, such as Cloudflare, which have more sophisticated
         # bot detection

From e9e7ee2c81fdf280f775da4e8a5cf0890b07f513 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:31:07 -0700
Subject: [PATCH 17/23] moved to only scanning 100 domains

---
 .github/workflows/ci.yml                      |    2 +-
 testing/top-1k.csv                            | 1000 -----------------
 testing/top-domains.csv                       |  101 ++
 ...ts.py => top_domains_integration_tests.py} |   19 +-
 4 files changed, 114 insertions(+), 1008 deletions(-)
 delete mode 100644 testing/top-1k.csv
 create mode 100644 testing/top-domains.csv
 rename testing/{top_1k_domains_integration_tests.py => top_domains_integration_tests.py} (85%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 61bf8b1f..bc7e4a46 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -57,4 +57,4 @@ jobs:
       - name: Large-Scale Integration Test
         run: |
           python --version
-          ./testing/top_1k_domains_integration_tests.py
+          ./testing/top_domains_integration_tests.py
diff --git a/testing/top-1k.csv b/testing/top-1k.csv
deleted file mode 100644
index 8d758930..00000000
--- a/testing/top-1k.csv
+++ /dev/null
@@ -1,1000 +0,0 @@
-m.fabswingers.com
-www.dmm.co.jp
-anoboy.baby
-www.cmoa.jp
-visortmo.com
-shahvani.com
-sattaamatka.co.com
-www.asos.com
-arca.live
-letterboxd.com
-www.ebay.com
-www.bfmtv.com
-www.rat.xxx
-my.xhtab3.com
-www.flashscore.com
-cz.pornhub.com
-studio.code.org
-www.t-online.de
-accounts.google.com
-asuratoon.com
-glife.bingoplus.com
-www.tiktok.com
-aniwave.to
-schools.madrasati.sa
-gall.dcinside.com
-m.fmkorea.com
-mangabuddy.com
-thisvid.com
-www.flipkart.com
-sp.oddspark.com
-www.fmkorea.com
-www.porn300.com
-context.reverso.net
-ijavhd.com
-www.mobile.de
-www.sexindrag.com
-de.m.wikipedia.org
-exhentai.org
-onlyindianx.cc
-secure.bankofamerica.com
-www.mercadolibre.com.ar
-www.linkedin.com
-detail.chiebukuro.yahoo.co.jp
-fr.pornhub.com
-m.indiamart.com
-www.ebay.de
-www.facebook.com
-www.researchgate.net
-battwo.com
-web.whatsapp.com
-pikabu.ru
-myreadingmanga.info
-www.netflix.com
-www.samsung.com
-www.tokopedia.com
-medium.com
-ge.xhamster.com
-www.bbc.com
-www.publi24.ro
-olimpbet.kz
-www.polovniautomobili.com
-m.yahoo.co.jp
-www.coco.gg
-www.69shu.pro
-yako.red
-escortbabylon.net
-poczta.wp.pl
-www.dagbladet.no
-m.ss.com
-m.bild.de
-ria.ru
-www.dailymotion.com
-retail.onlinesbi.sbi
-myactivity.google.com
-nregastrep.nic.in
-www.aznude.com
-coinmarketcap.com
-www.gazzetta.it
-tv.simontokx.online
-character.ai
-pmkisan.gov.in
-mobile-tracker-free.com
-quizizz.com
-mov.ibomma.rs
-www.masrawy.com
-www.google.es
-as.com
-m.youtube.com
-bpexch.com
-betinexchange.com
-ncz3u7cj2.com
-www.jusbrasil.com.br
-auto.drom.ru
-www.subito.it
-poki.com
-newtoki330.com
-giris.turkiye.gov.tr
-www.protothema.gr
-18comic.vip
-m.olx.pl
-tvn24.pl
-m.naver.com
-www.cardmarket.com
-wordwall.net
-www.varzesh3.com
-www.mynet.com
-es.aliexpress.com
-www.hentai.name
-www.hdhindisex.com
-www.netshoes.com.br
-wl4.peachavocado.com
-hackerdoslot.com
-www.hln.be
-xhdepot.site
-xvideosporno.blog.br
-czbooks.net
-genius.com
-ekstrabladet.dk
-launchpad.classlink.com
-danbooru.donmai.us
-www.action.com
-onlar.az
-doublelist.com
-ja.wikipedia.org
-news.livedoor.com
-www.anses.gob.ar
-mlbpark.donga.com
-support.google.com
-www.kidsa-z.com
-www.merkur.de
-accounts.spotify.com
-index.hu
-bollyflix.nexus
-www.hindustantimes.com
-web.spaggiari.eu
-scholar.google.com
-www.bestbuy.com
-www.quora.com
-www.instagram.com
-www.avito.ru
-www.sportskeeda.com
-www.xnxx.tv
-g1.globo.com
-www.daily.co.jp
-play.blooket.com
-simpcity.su
-www.kamababa.desi
-www.starfall.com
-nz.ua
-saednews.com
-www.finn.no
-member.sss.gov.ph
-hianime.to
-www.mangago.me
-e621.net
-ar.m.wikipedia.org
-gdz.ru
-byjus.com
-www.viva.co.id
-www.vinted.fr
-www.google.com.tr
-www.betpawa.ug
-www.sciencedirect.com
-brainly.co.id
-www.amazon.eg
-www.mercadolivre.com.br
-www.google.com.hk
-www.indeed.com
-www.lanacion.com.ar
-nl.pornhub.com
-steamcommunity.com
-an1.com
-mangakakalot.com
-truyenfull.vn
-linktr.ee
-m.cafe.daum.net
-vnexpress.net
-www.tgju.org
-www.ukr.net
-betproexch.com
-www.spotify.com
-fr.xhamster.com
-www.sugotoku.docomo.ne.jp
-kolkataff.in
-www.patria.org.ve
-www.roblox.com
-bdvenlinea.banvenez.com
-mail.google.com
-www.google.de
-news.infoseek.co.jp
-www.amazon.es
-www.espncricinfo.com
-zh.m.wikipedia.org
-www.aajtak.in
-n.news.naver.com
-www.livescore.com
-www.remove.bg
-fa.m.wikipedia.org
-secure.livechatinc.com
-www.almasryalyoum.com
-www.ebay.co.uk
-synergia.librus.pl
-www.ninisite.com
-www.google.pl
-chromewebstore.google.com
-m.olx.ua
-s.kakaku.com
-wimt.confirmtkt.com
-www.globo.com
-www.disneyplus.com
-es.pornhub.com
-gelbooru.com
-servicioscorp.anses.gob.ar
-www.xvideos3.com
-www.irs.gov
-xbato.com
-salonboard.com
-tel.ibomma.nexus
-tw.stock.yahoo.com
-www.tjk.org
-newtoki331.com
-readmanga.live
-www.programme-tv.net
-br.pinterest.com
-www.keralalotterytoday.com
-xxxbfvideos.org
-escortalligator.com.listcrawler.eu
-www.jutarnji.hr
-www.namasha.com
-finance.yahoo.com
-missav.com
-www.joyclub.de
-www.ign.com
-www.amazon.in
-www.independent.co.uk
-www.ryanair.com
-mangaschan.net
-mangatoto.org
-www.goojara.to
-www.tradingview.com
-www.twitch.tv
-11xplay.pro
-elpais.com
-snaptik.app
-people.com
-m.humoruniv.com
-ru.xhamster.com
-xhwide4.com
-www.inven.co.kr
-ssstik.io
-sso.acesso.gov.br
-www.cdiscount.com
-www.betika.com
-enabiz.gov.tr
-sattasport.in
-xoso.com.vn
-xnxx.health
-www.ynet.co.il
-www.google.com
-sattamatkano1.me
-hanime1.me
-novel18.syosetu.com
-www.dailymail.co.uk
-www.lefigaro.fr
-www.casinoplus.com.ph
-www.slideshare.net
-e-hentai.org
-m.tribunnews.com
-www.jlpt-maja.org.mm
-olxmewah.com
-www.futbin.com
-www.lxtgame.com
-www.wattpad.com
-music-fa.com
-www.iltalehti.fi
-www.google.com.mx
-timesofindia.indiatimes.com
-www.youjizz.com
-fantia.jp
-tools.usps.com
-www.aemet.es
-www.boyfriendtv.com
-jable.tv
-www.24h.com.vn
-faphouse.com
-www.list.am
-hot-sex-tube.com
-www.hurriyet.com.tr
-noodlemagazine.com
-www.xnxx3.com
-satta-king-fixed-no.in
-calendar.google.com
-www.3bmeteo.com
-coomer.su
-m.hqporner.com
-www.apornvideo.com
-dewafilm.xyz
-smt.docomo.ne.jp
-twitter.com
-account.samsung.com
-archiveofourown.org
-vlxx.moe
-m.sexemodel.com
-www.imagefap.com
-www.ojogodobicho.com
-www.dogdrip.net
-www.google.it
-finance.yahoo.co.jp
-www.aparat.com
-moi.ibomma.re
-pl.xhamster.com
-www.idnes.cz
-xhamster42.desi
-kompoz2.com
-tel.ibomma.rsvp
-m.vk.com
-www.amazon.fr
-www.cnnindonesia.com
-en.luxuretv.com
-select.mamastar.jp
-pubmed.ncbi.nlm.nih.gov
-www.biblegateway.com
-www.marmiton.org
-dantri.com.vn
-s.mogg.ameba.jp
-www.nikkansports.com
-v.daum.net
-www.avto.net
-www.naver.com
-m.hollywoodbets.net
-m.ok.ru
-www.ndtv.com
-rajwap.cc
-search.yahoo.co.jp
-ya.ru
-www.foxnews.com
-www.marca.com
-www.youtube.com
-www.bing.com
-xxxbfvideos.net
-www.nettruyenff.com
-zistop.com
-zbporn.tv
-www.unotv.com
-www.amazon.com
-sxyprn.com
-theqoo.net
-khoahoc.vietjack.com
-namu.wiki
-www.fulldesisex.com
-schools.emaktab.uz
-outlook.live.com
-www.discogs.com
-de.pornhub.org
-apps.facebook.com
-old.reddit.com
-www.flashscore.fr
-www.leroymerlin.fr
-www.jw.org
-www.aliexpress.com
-www.scribd.com
-bokepindoh.sbs
-duckduckgo.com
-drive.usercontent.google.com
-ecnavi.jp
-nypost.com
-www.amazon.co.jp
-www.ladepeche.fr
-www.patreon.com
-xhamster.desi
-game.granbluefantasy.jp
-www.google.co.in
-tsn.ua
-www.sexvid.xxx
-udn.com
-la.spankbang.com
-ge.xhamster.desi
-tamilyogi.red
-telegram.org
-www.otomoto.pl
-www.cnbcindonesia.com
-rule34.xxx
-www.bpexch.com
-topmanhua.com
-www.imdb.com
-xhtab3.com
-m.haber7.com
-porno18.site
-www.wp.pl
-stackoverflow.com
-www.sikayetvar.com
-actu.fr
-item.rakuten.co.jp
-produto.mercadolivre.com.br
-litnet.com
-grapee.jp
-myapps.classlink.com
-www.agoda.com
-www.index.hr
-beauty.hotpepper.jp
-mbigpara.hurriyet.com.tr
-kemono.su
-www.staseraintv.com
-shinigami.cx
-www.ouest-france.fr
-direct.lc.chat
-m.shein.com
-www.vg.no
-www.w3schools.com
-esaj.tjsp.jus.br
-hiroba.dpoint.docomo.ne.jp
-www.amazon.com.tr
-www.ozon.ru
-lichess.org
-www.goodreads.com
-www.mathplayground.com
-www.nexusmods.com
-app.hubspot.com
-funpay.com
-m.kooora.com
-www.wordreference.com
-www.goal.com
-truyenqqvn.com
-uk.indeed.com
-www.fanpage.it
-www.lowes.com
-www.diretta.it
-search.naver.com
-www.leboncoin.fr
-m.ppomppu.co.kr
-www.dns-shop.ru
-www.metropoles.com
-chaturbate.com
-www.serviporno.com
-bato.to
-game8.jp
-alarbda.com
-xxxvideo.link
-www.pinterest.com
-www.vietjack.com
-ita.xhamster.com
-www.trendyol.com
-www.ebay.it
-vegamovies.rsvp
-www.forbes.com
-return.gst.gov.in
-www.jcpenney.com
-xn--l3cg7a8a0cwa3f.com
-www.realtor.com
-ameblo.jp
-iporntv.net
-www.google.fr
-chance.enjoy.point.auone.jp
-business.facebook.com
-m.chaturbate.com
-www.5movierulz.dog
-id.xhamster.com
-weather.com
-www.chegg.com
-www.ixxx.com
-www.sportybet.com
-auto.ria.com
-m.youm7.com
-www.amazon.de
-mpokec.azet.sk
-www.filmyzilla.com.ng
-www.jra.go.jp
-in3x.net
-www.bt.dk
-www.thesun.co.uk
-it.m.wikipedia.org
-manhwaclan.com
-screenrant.com
-trilltrill.jp
-www.jiocinema.com
-www.tvonenews.com
-login.caixa.gov.br
-finance.naver.com
-www.bakecaincontrii.com
-emaktab.uz
-movie.eroterest.net
-www.1mg.com
-www.eltiempo.es
-tr.m.wikipedia.org
-tmohentai.com
-www.seznam.cz
-auth.uber.com
-photos.google.com
-www.flashscore.com.br
-kuji.dmarket.docomo.ne.jp
-docs.google.com
-www.amazon.com.mx
-sites.google.com
-cricbet99.win
-www.ilovepdf.com
-www.skroutz.gr
-my.porno365.pics
-support.microsoft.com
-freeip.me
-www.turkiye.gov.tr
-sports.yahoo.co.jp
-spboss.blog
-www.mundodeportivo.com
-www.tokyomotion.net
-fast.com
-www.conectate.com.do
-yatube.net
-www.chordtela.com
-br.skokka.com
-komikindo.tv
-www.gov.br
-www.elmundo.es
-app2dn.fextok.com
-race.sp.netkeiba.com
-evexxx.com
-www.clarin.com
-www.20minutos.es
-thisav.com
-www.ptt.cc
-v22-ispdy5ersd-ai-verify--gcs.g-xyz.xyz
-sports.news.naver.com
-sellercentral.amazon.com
-www.espn.com
-www.baidu.com
-zozo.jp
-www.n-tv.de
-dzen.ru
-harimanga.com
-lms.lausd.net
-m.happymh.com
-www.booking.com
-yts.mx
-gimy.ai
-pixabay.com
-tenki.jp
-gshow.globo.com
-me.porno666.link
-doujindesu.tv
-schools.kundelik.kz
-www.reddit.com
-www.zillow.com
-filmyfly.pm
-www.idealista.com
-www.binance.com
-www.interia.pl
-in.search.yahoo.com
-in.pinterest.com
-poop.com.co
-www.redtube.com
-www.mayoclinic.org
-betnacional.com
-br.betano.com
-www.1024tera.com
-9animetv.to
-www.jalan.net
-bakusai.com
-ko.xhamster2.com
-hochi.news
-vk.com
-www.coingecko.com
-www.oricon.co.jp
-moviesda9.me
-www.ajio.com
-www.ancestry.com
-www.credit-agricole.fr
-hdhub4u.pm
-client.schwab.com
-ncode.syosetu.com
-m.999.md
-www.istockphoto.com
-bbs.ruliweb.com
-www.costco.com
-www.curseforge.com
-search.rakuten.co.jp
-www.cricbuzz.com
-www.creditkarma.com
-www.sex.com
-baseball.yahoo.co.jp
-www.123-hd.com
-www.msn.com
-550909.com
-cashier.hkppay.com
-imgsrc.ru
-manga168.com
-xxxnd.com
-hitomi.la
-www.pussyboy.net
-piccoma.com
-m.autoplius.lt
-www.shaalaa.com
-email.seznam.cz
-pay.icespay.com
-www.superporn.com
-th.spankbang.com
-cn.pornhub.com
-www.desitelugusex.com
-yako.net
-www.freepik.com
-www.makemytrip.com
-store.steampowered.com
-okxxx1.com
-www.yahoo.com
-www.justdial.com
-www.udemy.com
-www.bet365.com
-www.merriam-webster.com
-discord.com
-sp.mbga.jp
-www.russianfood.com
-m.fanfiction.net
-www.kurir.rs
-www.zearn.org
-pet.gree-apps.net
-www.amarujala.com
-www.fiverr.com
-link.coupang.com
-clever.com
-lotterysambadresult.in
-www.clip2vip.com
-www.mediaexpert.pl
-pl.m.wikipedia.org
-www.letras.mus.br
-www.google.ca
-ifilo.net
-myaccount.google.com
-www.fuq.com
-www.eenadu.net
-m.magazineluiza.com.br
-www.google.co.uk
-faq.whatsapp.com
-in.indeed.com
-moviesda9.co
-mail.yahoo.com
-t.me
-m.tabor.ru
-news.google.com
-m.gsmarena.com
-onlyfans.com
-www.bilibili.tv
-wl2.peachavocado.com
-portal.auone.jp
-unifiedportal-mem.epfindia.gov.in
-uidai.gov.in
-www.flashscore.pl
-www.google.com.br
-m.beboo.ru
-login.yahoo.com
-tubidy.cool
-mail.ru
-www.etsy.com
-www.mediafire.com
-www.ilmeteo.it
-pl.pornhub.com
-transit.yahoo.co.jp
-chat.openai.com
-newtoki329.com
-www.cronista.com
-psiphon.news
-member.rakuten-sec.co.jp
-g.laroza.net
-skipthegames.com
-apnaludo.com
-www.xvideos53.com
-www.nu.nl
-www.sozcu.com.tr
-www.pussyspace.com
-m.pantip.com
-www.deviantart.com
-www.promiedos.com.ar
-www.homedepot.com
-www.tvguia.es
-www.wildberries.ru
-wol.jw.org
-rezka.ag
-www.macys.com
-www.buzzfeed.com
-www.dropbox.com
-hi.xhamster42.desi
-www.adobe.com
-www.coolmathgames.com
-rule34video.com
-www.google.co.jp
-cookpad.com
-ser.brstej.com
-togetter.com
-animeyuzu.com
-www.cityheaven.net
-www.furaffinity.net
-www.nikkei.com
-game.hiroba.dpoint.docomo.ne.jp
-www.argos.co.uk
-www.accuweather.com
-tw.yahoo.com
-br.xhamster.com
-www.ahindian.com
-liquipedia.net
-onedrive.live.com
-tathya.uidai.gov.in
-www.idealista.it
-www.pixiv.net
-testing.illuminateed.com
-uranai.nosv.org
-wl5.peachavocado.com
-hentailib.me
-www.chip.de
-www.redfin.com
-hot.noodlemagazine.com
-boards.4chan.org
-ru.sex-studentki.cash
-tukif.com
-kakuyomu.jp
-topics.smt.docomo.ne.jp
-jut.su
-www.xvideos.es
-www.zalando.de
-aniwatchtv.to
-uchi.ru
-www.fsiblog3.club
-www.yahoo.co.jp
-xnxx.com.se
-www.prokerala.com
-scratch.mit.edu
-www.emag.ro
-www.hdtube.porn
-meet.google.com
-m.search.naver.com
-www.bbc.co.uk
-xxxsexjav.com
-ficbook.net
-www.qorno.com
-github.com
-www.ikea.com
-www.target.com
-www.news.com.au
-www.haberler.com
-archive.org
-fr.m.wikipedia.org
-news.yahoo.co.jp
-mechacomic.jp
-www.terabox.app
-www.5movierulz.blog
-www.whatsapp.com
-cuetug.ntaonline.in
-www.walmart.com
-mangalib.me
-www.cnn.com
-xhamster2.com
-m.dcinside.com
-www.lemonde.fr
-www.universal-credit.service.gov.uk
-www.ad.nl
-www.annunci69.it
-m.ruliweb.com
-www.livescore.in
-www.xvideos4.com
-beta.character.ai
-s.dto.jp
-xxx18hot.com
-es.xhamster.com
-www.pornid.name
-www.moneycontrol.com
-av19.gg
-www3.nhk.or.jp
-news.goo.ne.jp
-www.sexybluefilm.com
-in.bookmyshow.com
-www.autotrader.co.uk
-www.telegraaf.nl
-www.newsweek.com
-www.sahibinden.com
-www.myntra.com
-gemini.google.com
-girlschannel.net
-hivemapper.com
-mat6tube.com
-ge.globo.com
-brainly.in
-de.pornhub.com
-www.filmweb.pl
-homepage.vivo.com
-diziwatch.net
-imhentai.xxx
-www.paypal.com
-www.kompas.com
-syosetu.org
-www.apple.com
-supjav.com
-www.kohls.com
-toyhou.se
-lectormanga.lat
-www.francetvinfo.fr
-www.xnxx2.com
-bet.caliente.mx
-bbs.animanch.com
-quillbot.com
-www.blic.rs
-www.digikala.com
-www.thscore.mobi
-ads.google.com
-vegamovies.ong
-www.gov.uk
-www.chess.com
-www.onet.pl
-shopee.com.br
-www.olx.com.br
-www.ixl.com
-www.ixbt.com
-www.ncbi.nlm.nih.gov
-dmarket.docomo.ne.jp
-articulo.mercadolibre.com.ar
-m.facebook.com
-sp.jra.jp
-xosodaiphat.com
-www.rezultati.com
-rt.pornhub.com
-www.52shuku.vip
-www.colamanga.com
-www.youporn.com
-motherless.com
-www.betway.co.za
-fapello.com
-kkstories.com
-mbasic.facebook.com
-fatalmodel.com
-www.hltv.org
-m.imdb.com
-www.bilibili.com
-www.nike.com
-www.amazon.com.br
-www.isaimini.com.sb
-www.pornhat.one
-www.milliyet.com.tr
-pfms.nic.in
-play.google.com
-www.livehindustan.com
-www.wannonce.com
-portal.librus.pl
-xhamster1.desi
-luxmovies.biz
-ru.m.wikipedia.org
-dpboss.services
-www.sexvid.pro
-xnxxcom.xyz
-www.croxyproxy.com
-pet.ibomma.bond
-www.xvideos.com
-drive.google.com
-pl.escort.club
-yaoilib.me
-znanija.com
-free.facebook.com
-wl1.peachavocado.com
-www.shutterstock.com
-www.eporner.com
-www2.hm.com
-newtoki328.com
-woman.excite.co.jp
-www.xvideos5.com
-br.search.yahoo.com
-spankbang.com
-www.gismeteo.ru
-www.uol.com.br
-fetlife.com
-serviziweb2.inps.it
-aternos.org
-wiadomosci.wp.pl
-www.speedtest.net
-gamewith.jp
-www.tuazar.com
-it.pornhub.com
-web.facebook.com
-www.erome.com
-www.screener.in
-client--gws-wiz-serp22-v1ddlcxru01-ioin.a-0.xyz
-services.gst.gov.in
-snapinsta.app
-www.xvideos2.com
-tw.news.yahoo.com
-one.google.com
-classroom.google.com
-happymod.com
-satta-king-fast.com
-remaker.ai
-www.ecosia.org
-video.hnext.jp
-pt.m.wikipedia.org
-open.spotify.com
-translate.google.com
-nhentai.net
-ww1.goojara.to
-m.site777.jp
-www.upwork.com
-www.terra.com.br
-www.sanook.com
-wapbold.net
-www.xnxx.com
-www.amazon.co.uk
-www.njuskalo.hr
-www.nairaland.com
-s.tabelog.com
-www.tbshop-ma.net
-m.cricbuzz.com
-www.amazon.ca
-www.expressen.se
-www.kleinanzeigen.de
-xxxhindi.to
-heylink.me
-www.dlsite.com
-www.hibiny.ru
-www.corriere.it
-www.nytimes.com
-xnxxsex.net
-www.ups.com
-rx.sp.mbga.jp
-brainly.com.br
-www.hepsiburada.com
-www.tripadvisor.com
-sportowefakty.wp.pl
-www.indianbfvideos.com
-82xnxx.com
-m.daum.net
-manhuascan.io
-www.mercadolibre.com.mx
-spankbang.party
-mangatoto.com
-en.wikipedia.org
-www.amazon.it
-persona.patria.org.ve
-www.tamildhool.net
-zinmanga.com
-www.otto.de
-91club06.com
-www.pornpics.com
-suumo.jp
-www.boatrace.jp
-www.infobae.com
-m.inven.co.kr
-quizlet.com
-turbo.az
-www.y2mate.com
-www.toppr.com
-studio.youtube.com
-m.avito.ru
-blog.livedoor.jp
-m.fabguys.com
-www.microsoft.com
-www.bol.com
-www.nitrotype.com
-www.temu.com
-goal.co
-www.khabaronline.ir
-91club.club
-allegro.pl
-funnyjunk.com
-auctions.yahoo.co.jp
-pt.pornhub.com
-lermanga.org
-rajshaladarpan.nic.in
-wto.to
-m.scan-manga.com
-www.canva.com
-play.contents.plus-msg.auone.jp
-ja.m.wikipedia.org
-www.carsensor.net
-www.nettruyenee.com
-zoom.us
-www.sarkariresult.com
-page.auctions.yahoo.co.jp
-shopee.co.id
-www.nesine.com
-www.wnacg.com
-www.dafont.com
-www.theguardian.com
-www.sinsay.com
-a24.app.gree-pf.net
-roboguru.ruangguru.com
-www.studocu.com
-www.timesbull.com
-yatv.pro
-nouen.sp.mbga.jp
-cardgames.io
-sp.pf.mbga.jp
-passbook.epfindia.gov.in
-pg.access-33.com
-wonporn.com
-yandex.com
-noutore.enjoy.point.auone.jp
-www.primevideo.com
-articulo.mercadolibre.com.mx
-www.pornhub.com
-www.rightmove.co.uk
-es.scribd.com
-jp.pornhub.com
-www.repubblica.it
-id.m.wikipedia.org
-m.skelbiu.lt
-www.marktplaats.nl
-es.m.wikipedia.org
-search.yahoo.com
-jp.spankbang.com
diff --git a/testing/top-domains.csv b/testing/top-domains.csv
new file mode 100644
index 00000000..f872d782
--- /dev/null
+++ b/testing/top-domains.csv
@@ -0,0 +1,101 @@
+m.fabswingers.com
+www.dmm.co.jp
+anoboy.baby
+www.cmoa.jp
+visortmo.com
+shahvani.com
+sattaamatka.co.com
+www.asos.com
+arca.live
+letterboxd.com
+www.ebay.com
+www.bfmtv.com
+www.rat.xxx
+my.xhtab3.com
+www.flashscore.com
+cz.pornhub.com
+studio.code.org
+www.t-online.de
+accounts.google.com
+asuratoon.com
+glife.bingoplus.com
+www.tiktok.com
+aniwave.to
+schools.madrasati.sa
+gall.dcinside.com
+m.fmkorea.com
+mangabuddy.com
+thisvid.com
+www.flipkart.com
+sp.oddspark.com
+www.fmkorea.com
+www.porn300.com
+context.reverso.net
+ijavhd.com
+www.mobile.de
+www.sexindrag.com
+de.m.wikipedia.org
+exhentai.org
+onlyindianx.cc
+secure.bankofamerica.com
+www.mercadolibre.com.ar
+www.linkedin.com
+detail.chiebukuro.yahoo.co.jp
+fr.pornhub.com
+m.indiamart.com
+www.ebay.de
+www.facebook.com
+www.researchgate.net
+battwo.com
+web.whatsapp.com
+pikabu.ru
+myreadingmanga.info
+www.netflix.com
+www.samsung.com
+www.tokopedia.com
+medium.com
+ge.xhamster.com
+www.bbc.com
+www.publi24.ro
+olimpbet.kz
+www.polovniautomobili.com
+m.yahoo.co.jp
+www.coco.gg
+www.69shu.pro
+yako.red
+escortbabylon.net
+poczta.wp.pl
+www.dagbladet.no
+m.ss.com
+m.bild.de
+ria.ru
+www.dailymotion.com
+retail.onlinesbi.sbi
+myactivity.google.com
+nregastrep.nic.in
+www.aznude.com
+coinmarketcap.com
+www.gazzetta.it
+tv.simontokx.online
+character.ai
+pmkisan.gov.in
+mobile-tracker-free.com
+quizizz.com
+mov.ibomma.rs
+www.masrawy.com
+www.google.es
+as.com
+m.youtube.com
+bpexch.com
+betinexchange.com
+ncz3u7cj2.com
+www.jusbrasil.com.br
+auto.drom.ru
+www.subito.it
+poki.com
+newtoki330.com
+giris.turkiye.gov.tr
+www.protothema.gr
+18comic.vip
+m.olx.pl
+tvn24.pl
\ No newline at end of file
diff --git a/testing/top_1k_domains_integration_tests.py b/testing/top_domains_integration_tests.py
similarity index 85%
rename from testing/top_1k_domains_integration_tests.py
rename to testing/top_domains_integration_tests.py
index 6f0fb08f..9a61e960 100755
--- a/testing/top_1k_domains_integration_tests.py
+++ b/testing/top_domains_integration_tests.py
@@ -11,7 +11,7 @@
 import json
 
 ZDNS_EXECUTABLE = "./zdns"
-TOP_1K_DOMAINS_FILE = "./testing/top-1k.csv"
+TOP_DOMAINS_FILE = "./testing/top-domains.csv"
 
 def can_request_successfully(domain: str, ip: str) -> bool:
     user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15"
@@ -141,7 +141,7 @@ def get_zdns_results_a_lookup(domains: List[str]) -> List[tuple[str, str]]:
 class TestZDNS(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        with open(TOP_1K_DOMAINS_FILE, "r") as f:
+        with open(TOP_DOMAINS_FILE, "r") as f:
             domains = f.read().splitlines()
         # These are those domains which we can successfully request using the requests library directed at an IP address
         # This excludes domains that use some form of DDoS mitigation, such as Cloudflare, which have more sophisticated
@@ -151,15 +151,20 @@ def setUpClass(cls):
     def test_zdns_a(self):
         zdns = get_zdns_results_a(self.known_reachable_domains)
         print(f"ZDNS resolved {len(zdns)} domains to IP addresses from request-able domains")
-        for domain, ip in zdns:
-            self.assertTrue(can_request_successfully(domain, ip),
-                            "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
+        # Check that the IP addresses ZDNS resolved the domains to are reachable, but in parallel
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            results = [executor.submit(can_request_successfully, domain, ip) for domain, ip in zdns]
+            for future in concurrent.futures.as_completed(results):
+                self.assertTrue(future.result(), "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
 
     def test_zdns_a_lookup(self):
         zdns = get_zdns_results_a_lookup(self.known_reachable_domains)
         print(f"ZDNS resolved {len(zdns)} domains to IP addresses from request-able domains")
-        for domain, ip in zdns:
-            self.assertTrue(can_request_successfully(domain, ip), "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
+        # Check that the IP addresses ZDNS resolved the domains to are reachable, but in parallel
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            results = [executor.submit(can_request_successfully, domain, ip) for domain, ip in zdns]
+            for future in concurrent.futures.as_completed(results):
+                self.assertTrue(future.result(), "ZDNS resolved a domain to an IP address that will not respond to requests for the given domain")
 
 
 if __name__ == '__main__':

From 581fa709751968b1c5ae1c382e5385934a698a1c Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:39:05 -0700
Subject: [PATCH 18/23] cleanup

---
 testing/{top-domains.csv => domains.csv} |  2 +-
 testing/top_domains_integration_tests.py | 10 +++-------
 2 files changed, 4 insertions(+), 8 deletions(-)
 rename testing/{top-domains.csv => domains.csv} (99%)

diff --git a/testing/top-domains.csv b/testing/domains.csv
similarity index 99%
rename from testing/top-domains.csv
rename to testing/domains.csv
index f872d782..816c0313 100644
--- a/testing/top-domains.csv
+++ b/testing/domains.csv
@@ -98,4 +98,4 @@ giris.turkiye.gov.tr
 www.protothema.gr
 18comic.vip
 m.olx.pl
-tvn24.pl
\ No newline at end of file
+tvn24.pl
diff --git a/testing/top_domains_integration_tests.py b/testing/top_domains_integration_tests.py
index 9a61e960..2cf76f35 100755
--- a/testing/top_domains_integration_tests.py
+++ b/testing/top_domains_integration_tests.py
@@ -11,8 +11,9 @@
 import json
 
 ZDNS_EXECUTABLE = "./zdns"
-TOP_DOMAINS_FILE = "./testing/top-domains.csv"
+TOP_DOMAINS_FILE = "./testing/domains.csv"
 
+# This function checks if a domain can be successfully requested at a given IP
 def can_request_successfully(domain: str, ip: str) -> bool:
     user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15"
     headers = {"Host": domain, "User-Agent": user_agent}
@@ -91,16 +92,11 @@ def get_requestable_domains(domains: List[str]) -> List[str]:
         for future in concurrent.futures.as_completed(results):
             domain, ip, result = future.result()
             if result == 0:
+                # domain is reachable
                 successes_requests.append(domain)
-            # elif result == 1:
-            #     successes_puppeteer.append(domain)
-            elif result == -1:
-                print(f"Failed to visit {domain} with either requests or puppeteer")
 
     print(f"Successfully requested {len(successes_requests)} out of {len(domains)} domains with requests")
-    # print(f"Successfully visited {len(successes_puppeteer)} out of {len(domains)} domains with puppeteer")
     print("Requests successes:", successes_requests)
-    # print("Puppeteer successes:", successes_puppeteer)
     print("Failures:", set(domains) - set(successes_requests))
     return successes_requests
 

From 4b04b9f882a0fdb70a1b622b856d11fa4b98acc3 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:40:53 -0700
Subject: [PATCH 19/23] fixed name of CI action

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bc7e4a46..7ceaa417 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -41,7 +41,7 @@ jobs:
           sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
           python --version
           ./testing/integration_tests.py
-  build-and-test-top-1k:
+  build-and-test-large-scale:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

From fc889fda69a28f20a0bde93804e475ff1f8d233f Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:42:09 -0700
Subject: [PATCH 20/23] added rest of domains

---
 testing/domains.csv | 899 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 899 insertions(+)

diff --git a/testing/domains.csv b/testing/domains.csv
index 816c0313..8d758930 100644
--- a/testing/domains.csv
+++ b/testing/domains.csv
@@ -99,3 +99,902 @@ www.protothema.gr
 18comic.vip
 m.olx.pl
 tvn24.pl
+m.naver.com
+www.cardmarket.com
+wordwall.net
+www.varzesh3.com
+www.mynet.com
+es.aliexpress.com
+www.hentai.name
+www.hdhindisex.com
+www.netshoes.com.br
+wl4.peachavocado.com
+hackerdoslot.com
+www.hln.be
+xhdepot.site
+xvideosporno.blog.br
+czbooks.net
+genius.com
+ekstrabladet.dk
+launchpad.classlink.com
+danbooru.donmai.us
+www.action.com
+onlar.az
+doublelist.com
+ja.wikipedia.org
+news.livedoor.com
+www.anses.gob.ar
+mlbpark.donga.com
+support.google.com
+www.kidsa-z.com
+www.merkur.de
+accounts.spotify.com
+index.hu
+bollyflix.nexus
+www.hindustantimes.com
+web.spaggiari.eu
+scholar.google.com
+www.bestbuy.com
+www.quora.com
+www.instagram.com
+www.avito.ru
+www.sportskeeda.com
+www.xnxx.tv
+g1.globo.com
+www.daily.co.jp
+play.blooket.com
+simpcity.su
+www.kamababa.desi
+www.starfall.com
+nz.ua
+saednews.com
+www.finn.no
+member.sss.gov.ph
+hianime.to
+www.mangago.me
+e621.net
+ar.m.wikipedia.org
+gdz.ru
+byjus.com
+www.viva.co.id
+www.vinted.fr
+www.google.com.tr
+www.betpawa.ug
+www.sciencedirect.com
+brainly.co.id
+www.amazon.eg
+www.mercadolivre.com.br
+www.google.com.hk
+www.indeed.com
+www.lanacion.com.ar
+nl.pornhub.com
+steamcommunity.com
+an1.com
+mangakakalot.com
+truyenfull.vn
+linktr.ee
+m.cafe.daum.net
+vnexpress.net
+www.tgju.org
+www.ukr.net
+betproexch.com
+www.spotify.com
+fr.xhamster.com
+www.sugotoku.docomo.ne.jp
+kolkataff.in
+www.patria.org.ve
+www.roblox.com
+bdvenlinea.banvenez.com
+mail.google.com
+www.google.de
+news.infoseek.co.jp
+www.amazon.es
+www.espncricinfo.com
+zh.m.wikipedia.org
+www.aajtak.in
+n.news.naver.com
+www.livescore.com
+www.remove.bg
+fa.m.wikipedia.org
+secure.livechatinc.com
+www.almasryalyoum.com
+www.ebay.co.uk
+synergia.librus.pl
+www.ninisite.com
+www.google.pl
+chromewebstore.google.com
+m.olx.ua
+s.kakaku.com
+wimt.confirmtkt.com
+www.globo.com
+www.disneyplus.com
+es.pornhub.com
+gelbooru.com
+servicioscorp.anses.gob.ar
+www.xvideos3.com
+www.irs.gov
+xbato.com
+salonboard.com
+tel.ibomma.nexus
+tw.stock.yahoo.com
+www.tjk.org
+newtoki331.com
+readmanga.live
+www.programme-tv.net
+br.pinterest.com
+www.keralalotterytoday.com
+xxxbfvideos.org
+escortalligator.com.listcrawler.eu
+www.jutarnji.hr
+www.namasha.com
+finance.yahoo.com
+missav.com
+www.joyclub.de
+www.ign.com
+www.amazon.in
+www.independent.co.uk
+www.ryanair.com
+mangaschan.net
+mangatoto.org
+www.goojara.to
+www.tradingview.com
+www.twitch.tv
+11xplay.pro
+elpais.com
+snaptik.app
+people.com
+m.humoruniv.com
+ru.xhamster.com
+xhwide4.com
+www.inven.co.kr
+ssstik.io
+sso.acesso.gov.br
+www.cdiscount.com
+www.betika.com
+enabiz.gov.tr
+sattasport.in
+xoso.com.vn
+xnxx.health
+www.ynet.co.il
+www.google.com
+sattamatkano1.me
+hanime1.me
+novel18.syosetu.com
+www.dailymail.co.uk
+www.lefigaro.fr
+www.casinoplus.com.ph
+www.slideshare.net
+e-hentai.org
+m.tribunnews.com
+www.jlpt-maja.org.mm
+olxmewah.com
+www.futbin.com
+www.lxtgame.com
+www.wattpad.com
+music-fa.com
+www.iltalehti.fi
+www.google.com.mx
+timesofindia.indiatimes.com
+www.youjizz.com
+fantia.jp
+tools.usps.com
+www.aemet.es
+www.boyfriendtv.com
+jable.tv
+www.24h.com.vn
+faphouse.com
+www.list.am
+hot-sex-tube.com
+www.hurriyet.com.tr
+noodlemagazine.com
+www.xnxx3.com
+satta-king-fixed-no.in
+calendar.google.com
+www.3bmeteo.com
+coomer.su
+m.hqporner.com
+www.apornvideo.com
+dewafilm.xyz
+smt.docomo.ne.jp
+twitter.com
+account.samsung.com
+archiveofourown.org
+vlxx.moe
+m.sexemodel.com
+www.imagefap.com
+www.ojogodobicho.com
+www.dogdrip.net
+www.google.it
+finance.yahoo.co.jp
+www.aparat.com
+moi.ibomma.re
+pl.xhamster.com
+www.idnes.cz
+xhamster42.desi
+kompoz2.com
+tel.ibomma.rsvp
+m.vk.com
+www.amazon.fr
+www.cnnindonesia.com
+en.luxuretv.com
+select.mamastar.jp
+pubmed.ncbi.nlm.nih.gov
+www.biblegateway.com
+www.marmiton.org
+dantri.com.vn
+s.mogg.ameba.jp
+www.nikkansports.com
+v.daum.net
+www.avto.net
+www.naver.com
+m.hollywoodbets.net
+m.ok.ru
+www.ndtv.com
+rajwap.cc
+search.yahoo.co.jp
+ya.ru
+www.foxnews.com
+www.marca.com
+www.youtube.com
+www.bing.com
+xxxbfvideos.net
+www.nettruyenff.com
+zistop.com
+zbporn.tv
+www.unotv.com
+www.amazon.com
+sxyprn.com
+theqoo.net
+khoahoc.vietjack.com
+namu.wiki
+www.fulldesisex.com
+schools.emaktab.uz
+outlook.live.com
+www.discogs.com
+de.pornhub.org
+apps.facebook.com
+old.reddit.com
+www.flashscore.fr
+www.leroymerlin.fr
+www.jw.org
+www.aliexpress.com
+www.scribd.com
+bokepindoh.sbs
+duckduckgo.com
+drive.usercontent.google.com
+ecnavi.jp
+nypost.com
+www.amazon.co.jp
+www.ladepeche.fr
+www.patreon.com
+xhamster.desi
+game.granbluefantasy.jp
+www.google.co.in
+tsn.ua
+www.sexvid.xxx
+udn.com
+la.spankbang.com
+ge.xhamster.desi
+tamilyogi.red
+telegram.org
+www.otomoto.pl
+www.cnbcindonesia.com
+rule34.xxx
+www.bpexch.com
+topmanhua.com
+www.imdb.com
+xhtab3.com
+m.haber7.com
+porno18.site
+www.wp.pl
+stackoverflow.com
+www.sikayetvar.com
+actu.fr
+item.rakuten.co.jp
+produto.mercadolivre.com.br
+litnet.com
+grapee.jp
+myapps.classlink.com
+www.agoda.com
+www.index.hr
+beauty.hotpepper.jp
+mbigpara.hurriyet.com.tr
+kemono.su
+www.staseraintv.com
+shinigami.cx
+www.ouest-france.fr
+direct.lc.chat
+m.shein.com
+www.vg.no
+www.w3schools.com
+esaj.tjsp.jus.br
+hiroba.dpoint.docomo.ne.jp
+www.amazon.com.tr
+www.ozon.ru
+lichess.org
+www.goodreads.com
+www.mathplayground.com
+www.nexusmods.com
+app.hubspot.com
+funpay.com
+m.kooora.com
+www.wordreference.com
+www.goal.com
+truyenqqvn.com
+uk.indeed.com
+www.fanpage.it
+www.lowes.com
+www.diretta.it
+search.naver.com
+www.leboncoin.fr
+m.ppomppu.co.kr
+www.dns-shop.ru
+www.metropoles.com
+chaturbate.com
+www.serviporno.com
+bato.to
+game8.jp
+alarbda.com
+xxxvideo.link
+www.pinterest.com
+www.vietjack.com
+ita.xhamster.com
+www.trendyol.com
+www.ebay.it
+vegamovies.rsvp
+www.forbes.com
+return.gst.gov.in
+www.jcpenney.com
+xn--l3cg7a8a0cwa3f.com
+www.realtor.com
+ameblo.jp
+iporntv.net
+www.google.fr
+chance.enjoy.point.auone.jp
+business.facebook.com
+m.chaturbate.com
+www.5movierulz.dog
+id.xhamster.com
+weather.com
+www.chegg.com
+www.ixxx.com
+www.sportybet.com
+auto.ria.com
+m.youm7.com
+www.amazon.de
+mpokec.azet.sk
+www.filmyzilla.com.ng
+www.jra.go.jp
+in3x.net
+www.bt.dk
+www.thesun.co.uk
+it.m.wikipedia.org
+manhwaclan.com
+screenrant.com
+trilltrill.jp
+www.jiocinema.com
+www.tvonenews.com
+login.caixa.gov.br
+finance.naver.com
+www.bakecaincontrii.com
+emaktab.uz
+movie.eroterest.net
+www.1mg.com
+www.eltiempo.es
+tr.m.wikipedia.org
+tmohentai.com
+www.seznam.cz
+auth.uber.com
+photos.google.com
+www.flashscore.com.br
+kuji.dmarket.docomo.ne.jp
+docs.google.com
+www.amazon.com.mx
+sites.google.com
+cricbet99.win
+www.ilovepdf.com
+www.skroutz.gr
+my.porno365.pics
+support.microsoft.com
+freeip.me
+www.turkiye.gov.tr
+sports.yahoo.co.jp
+spboss.blog
+www.mundodeportivo.com
+www.tokyomotion.net
+fast.com
+www.conectate.com.do
+yatube.net
+www.chordtela.com
+br.skokka.com
+komikindo.tv
+www.gov.br
+www.elmundo.es
+app2dn.fextok.com
+race.sp.netkeiba.com
+evexxx.com
+www.clarin.com
+www.20minutos.es
+thisav.com
+www.ptt.cc
+v22-ispdy5ersd-ai-verify--gcs.g-xyz.xyz
+sports.news.naver.com
+sellercentral.amazon.com
+www.espn.com
+www.baidu.com
+zozo.jp
+www.n-tv.de
+dzen.ru
+harimanga.com
+lms.lausd.net
+m.happymh.com
+www.booking.com
+yts.mx
+gimy.ai
+pixabay.com
+tenki.jp
+gshow.globo.com
+me.porno666.link
+doujindesu.tv
+schools.kundelik.kz
+www.reddit.com
+www.zillow.com
+filmyfly.pm
+www.idealista.com
+www.binance.com
+www.interia.pl
+in.search.yahoo.com
+in.pinterest.com
+poop.com.co
+www.redtube.com
+www.mayoclinic.org
+betnacional.com
+br.betano.com
+www.1024tera.com
+9animetv.to
+www.jalan.net
+bakusai.com
+ko.xhamster2.com
+hochi.news
+vk.com
+www.coingecko.com
+www.oricon.co.jp
+moviesda9.me
+www.ajio.com
+www.ancestry.com
+www.credit-agricole.fr
+hdhub4u.pm
+client.schwab.com
+ncode.syosetu.com
+m.999.md
+www.istockphoto.com
+bbs.ruliweb.com
+www.costco.com
+www.curseforge.com
+search.rakuten.co.jp
+www.cricbuzz.com
+www.creditkarma.com
+www.sex.com
+baseball.yahoo.co.jp
+www.123-hd.com
+www.msn.com
+550909.com
+cashier.hkppay.com
+imgsrc.ru
+manga168.com
+xxxnd.com
+hitomi.la
+www.pussyboy.net
+piccoma.com
+m.autoplius.lt
+www.shaalaa.com
+email.seznam.cz
+pay.icespay.com
+www.superporn.com
+th.spankbang.com
+cn.pornhub.com
+www.desitelugusex.com
+yako.net
+www.freepik.com
+www.makemytrip.com
+store.steampowered.com
+okxxx1.com
+www.yahoo.com
+www.justdial.com
+www.udemy.com
+www.bet365.com
+www.merriam-webster.com
+discord.com
+sp.mbga.jp
+www.russianfood.com
+m.fanfiction.net
+www.kurir.rs
+www.zearn.org
+pet.gree-apps.net
+www.amarujala.com
+www.fiverr.com
+link.coupang.com
+clever.com
+lotterysambadresult.in
+www.clip2vip.com
+www.mediaexpert.pl
+pl.m.wikipedia.org
+www.letras.mus.br
+www.google.ca
+ifilo.net
+myaccount.google.com
+www.fuq.com
+www.eenadu.net
+m.magazineluiza.com.br
+www.google.co.uk
+faq.whatsapp.com
+in.indeed.com
+moviesda9.co
+mail.yahoo.com
+t.me
+m.tabor.ru
+news.google.com
+m.gsmarena.com
+onlyfans.com
+www.bilibili.tv
+wl2.peachavocado.com
+portal.auone.jp
+unifiedportal-mem.epfindia.gov.in
+uidai.gov.in
+www.flashscore.pl
+www.google.com.br
+m.beboo.ru
+login.yahoo.com
+tubidy.cool
+mail.ru
+www.etsy.com
+www.mediafire.com
+www.ilmeteo.it
+pl.pornhub.com
+transit.yahoo.co.jp
+chat.openai.com
+newtoki329.com
+www.cronista.com
+psiphon.news
+member.rakuten-sec.co.jp
+g.laroza.net
+skipthegames.com
+apnaludo.com
+www.xvideos53.com
+www.nu.nl
+www.sozcu.com.tr
+www.pussyspace.com
+m.pantip.com
+www.deviantart.com
+www.promiedos.com.ar
+www.homedepot.com
+www.tvguia.es
+www.wildberries.ru
+wol.jw.org
+rezka.ag
+www.macys.com
+www.buzzfeed.com
+www.dropbox.com
+hi.xhamster42.desi
+www.adobe.com
+www.coolmathgames.com
+rule34video.com
+www.google.co.jp
+cookpad.com
+ser.brstej.com
+togetter.com
+animeyuzu.com
+www.cityheaven.net
+www.furaffinity.net
+www.nikkei.com
+game.hiroba.dpoint.docomo.ne.jp
+www.argos.co.uk
+www.accuweather.com
+tw.yahoo.com
+br.xhamster.com
+www.ahindian.com
+liquipedia.net
+onedrive.live.com
+tathya.uidai.gov.in
+www.idealista.it
+www.pixiv.net
+testing.illuminateed.com
+uranai.nosv.org
+wl5.peachavocado.com
+hentailib.me
+www.chip.de
+www.redfin.com
+hot.noodlemagazine.com
+boards.4chan.org
+ru.sex-studentki.cash
+tukif.com
+kakuyomu.jp
+topics.smt.docomo.ne.jp
+jut.su
+www.xvideos.es
+www.zalando.de
+aniwatchtv.to
+uchi.ru
+www.fsiblog3.club
+www.yahoo.co.jp
+xnxx.com.se
+www.prokerala.com
+scratch.mit.edu
+www.emag.ro
+www.hdtube.porn
+meet.google.com
+m.search.naver.com
+www.bbc.co.uk
+xxxsexjav.com
+ficbook.net
+www.qorno.com
+github.com
+www.ikea.com
+www.target.com
+www.news.com.au
+www.haberler.com
+archive.org
+fr.m.wikipedia.org
+news.yahoo.co.jp
+mechacomic.jp
+www.terabox.app
+www.5movierulz.blog
+www.whatsapp.com
+cuetug.ntaonline.in
+www.walmart.com
+mangalib.me
+www.cnn.com
+xhamster2.com
+m.dcinside.com
+www.lemonde.fr
+www.universal-credit.service.gov.uk
+www.ad.nl
+www.annunci69.it
+m.ruliweb.com
+www.livescore.in
+www.xvideos4.com
+beta.character.ai
+s.dto.jp
+xxx18hot.com
+es.xhamster.com
+www.pornid.name
+www.moneycontrol.com
+av19.gg
+www3.nhk.or.jp
+news.goo.ne.jp
+www.sexybluefilm.com
+in.bookmyshow.com
+www.autotrader.co.uk
+www.telegraaf.nl
+www.newsweek.com
+www.sahibinden.com
+www.myntra.com
+gemini.google.com
+girlschannel.net
+hivemapper.com
+mat6tube.com
+ge.globo.com
+brainly.in
+de.pornhub.com
+www.filmweb.pl
+homepage.vivo.com
+diziwatch.net
+imhentai.xxx
+www.paypal.com
+www.kompas.com
+syosetu.org
+www.apple.com
+supjav.com
+www.kohls.com
+toyhou.se
+lectormanga.lat
+www.francetvinfo.fr
+www.xnxx2.com
+bet.caliente.mx
+bbs.animanch.com
+quillbot.com
+www.blic.rs
+www.digikala.com
+www.thscore.mobi
+ads.google.com
+vegamovies.ong
+www.gov.uk
+www.chess.com
+www.onet.pl
+shopee.com.br
+www.olx.com.br
+www.ixl.com
+www.ixbt.com
+www.ncbi.nlm.nih.gov
+dmarket.docomo.ne.jp
+articulo.mercadolibre.com.ar
+m.facebook.com
+sp.jra.jp
+xosodaiphat.com
+www.rezultati.com
+rt.pornhub.com
+www.52shuku.vip
+www.colamanga.com
+www.youporn.com
+motherless.com
+www.betway.co.za
+fapello.com
+kkstories.com
+mbasic.facebook.com
+fatalmodel.com
+www.hltv.org
+m.imdb.com
+www.bilibili.com
+www.nike.com
+www.amazon.com.br
+www.isaimini.com.sb
+www.pornhat.one
+www.milliyet.com.tr
+pfms.nic.in
+play.google.com
+www.livehindustan.com
+www.wannonce.com
+portal.librus.pl
+xhamster1.desi
+luxmovies.biz
+ru.m.wikipedia.org
+dpboss.services
+www.sexvid.pro
+xnxxcom.xyz
+www.croxyproxy.com
+pet.ibomma.bond
+www.xvideos.com
+drive.google.com
+pl.escort.club
+yaoilib.me
+znanija.com
+free.facebook.com
+wl1.peachavocado.com
+www.shutterstock.com
+www.eporner.com
+www2.hm.com
+newtoki328.com
+woman.excite.co.jp
+www.xvideos5.com
+br.search.yahoo.com
+spankbang.com
+www.gismeteo.ru
+www.uol.com.br
+fetlife.com
+serviziweb2.inps.it
+aternos.org
+wiadomosci.wp.pl
+www.speedtest.net
+gamewith.jp
+www.tuazar.com
+it.pornhub.com
+web.facebook.com
+www.erome.com
+www.screener.in
+client--gws-wiz-serp22-v1ddlcxru01-ioin.a-0.xyz
+services.gst.gov.in
+snapinsta.app
+www.xvideos2.com
+tw.news.yahoo.com
+one.google.com
+classroom.google.com
+happymod.com
+satta-king-fast.com
+remaker.ai
+www.ecosia.org
+video.hnext.jp
+pt.m.wikipedia.org
+open.spotify.com
+translate.google.com
+nhentai.net
+ww1.goojara.to
+m.site777.jp
+www.upwork.com
+www.terra.com.br
+www.sanook.com
+wapbold.net
+www.xnxx.com
+www.amazon.co.uk
+www.njuskalo.hr
+www.nairaland.com
+s.tabelog.com
+www.tbshop-ma.net
+m.cricbuzz.com
+www.amazon.ca
+www.expressen.se
+www.kleinanzeigen.de
+xxxhindi.to
+heylink.me
+www.dlsite.com
+www.hibiny.ru
+www.corriere.it
+www.nytimes.com
+xnxxsex.net
+www.ups.com
+rx.sp.mbga.jp
+brainly.com.br
+www.hepsiburada.com
+www.tripadvisor.com
+sportowefakty.wp.pl
+www.indianbfvideos.com
+82xnxx.com
+m.daum.net
+manhuascan.io
+www.mercadolibre.com.mx
+spankbang.party
+mangatoto.com
+en.wikipedia.org
+www.amazon.it
+persona.patria.org.ve
+www.tamildhool.net
+zinmanga.com
+www.otto.de
+91club06.com
+www.pornpics.com
+suumo.jp
+www.boatrace.jp
+www.infobae.com
+m.inven.co.kr
+quizlet.com
+turbo.az
+www.y2mate.com
+www.toppr.com
+studio.youtube.com
+m.avito.ru
+blog.livedoor.jp
+m.fabguys.com
+www.microsoft.com
+www.bol.com
+www.nitrotype.com
+www.temu.com
+goal.co
+www.khabaronline.ir
+91club.club
+allegro.pl
+funnyjunk.com
+auctions.yahoo.co.jp
+pt.pornhub.com
+lermanga.org
+rajshaladarpan.nic.in
+wto.to
+m.scan-manga.com
+www.canva.com
+play.contents.plus-msg.auone.jp
+ja.m.wikipedia.org
+www.carsensor.net
+www.nettruyenee.com
+zoom.us
+www.sarkariresult.com
+page.auctions.yahoo.co.jp
+shopee.co.id
+www.nesine.com
+www.wnacg.com
+www.dafont.com
+www.theguardian.com
+www.sinsay.com
+a24.app.gree-pf.net
+roboguru.ruangguru.com
+www.studocu.com
+www.timesbull.com
+yatv.pro
+nouen.sp.mbga.jp
+cardgames.io
+sp.pf.mbga.jp
+passbook.epfindia.gov.in
+pg.access-33.com
+wonporn.com
+yandex.com
+noutore.enjoy.point.auone.jp
+www.primevideo.com
+articulo.mercadolibre.com.mx
+www.pornhub.com
+www.rightmove.co.uk
+es.scribd.com
+jp.pornhub.com
+www.repubblica.it
+id.m.wikipedia.org
+m.skelbiu.lt
+www.marktplaats.nl
+es.m.wikipedia.org
+search.yahoo.com
+jp.spankbang.com

From 1b87244e7e89b9f2095650b27bd33328a3a3ec2c Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:47:44 -0700
Subject: [PATCH 21/23] moved to 500 domains

---
 testing/domains.csv | 500 --------------------------------------------
 1 file changed, 500 deletions(-)

diff --git a/testing/domains.csv b/testing/domains.csv
index 8d758930..741b4200 100644
--- a/testing/domains.csv
+++ b/testing/domains.csv
@@ -498,503 +498,3 @@ my.porno365.pics
 support.microsoft.com
 freeip.me
 www.turkiye.gov.tr
-sports.yahoo.co.jp
-spboss.blog
-www.mundodeportivo.com
-www.tokyomotion.net
-fast.com
-www.conectate.com.do
-yatube.net
-www.chordtela.com
-br.skokka.com
-komikindo.tv
-www.gov.br
-www.elmundo.es
-app2dn.fextok.com
-race.sp.netkeiba.com
-evexxx.com
-www.clarin.com
-www.20minutos.es
-thisav.com
-www.ptt.cc
-v22-ispdy5ersd-ai-verify--gcs.g-xyz.xyz
-sports.news.naver.com
-sellercentral.amazon.com
-www.espn.com
-www.baidu.com
-zozo.jp
-www.n-tv.de
-dzen.ru
-harimanga.com
-lms.lausd.net
-m.happymh.com
-www.booking.com
-yts.mx
-gimy.ai
-pixabay.com
-tenki.jp
-gshow.globo.com
-me.porno666.link
-doujindesu.tv
-schools.kundelik.kz
-www.reddit.com
-www.zillow.com
-filmyfly.pm
-www.idealista.com
-www.binance.com
-www.interia.pl
-in.search.yahoo.com
-in.pinterest.com
-poop.com.co
-www.redtube.com
-www.mayoclinic.org
-betnacional.com
-br.betano.com
-www.1024tera.com
-9animetv.to
-www.jalan.net
-bakusai.com
-ko.xhamster2.com
-hochi.news
-vk.com
-www.coingecko.com
-www.oricon.co.jp
-moviesda9.me
-www.ajio.com
-www.ancestry.com
-www.credit-agricole.fr
-hdhub4u.pm
-client.schwab.com
-ncode.syosetu.com
-m.999.md
-www.istockphoto.com
-bbs.ruliweb.com
-www.costco.com
-www.curseforge.com
-search.rakuten.co.jp
-www.cricbuzz.com
-www.creditkarma.com
-www.sex.com
-baseball.yahoo.co.jp
-www.123-hd.com
-www.msn.com
-550909.com
-cashier.hkppay.com
-imgsrc.ru
-manga168.com
-xxxnd.com
-hitomi.la
-www.pussyboy.net
-piccoma.com
-m.autoplius.lt
-www.shaalaa.com
-email.seznam.cz
-pay.icespay.com
-www.superporn.com
-th.spankbang.com
-cn.pornhub.com
-www.desitelugusex.com
-yako.net
-www.freepik.com
-www.makemytrip.com
-store.steampowered.com
-okxxx1.com
-www.yahoo.com
-www.justdial.com
-www.udemy.com
-www.bet365.com
-www.merriam-webster.com
-discord.com
-sp.mbga.jp
-www.russianfood.com
-m.fanfiction.net
-www.kurir.rs
-www.zearn.org
-pet.gree-apps.net
-www.amarujala.com
-www.fiverr.com
-link.coupang.com
-clever.com
-lotterysambadresult.in
-www.clip2vip.com
-www.mediaexpert.pl
-pl.m.wikipedia.org
-www.letras.mus.br
-www.google.ca
-ifilo.net
-myaccount.google.com
-www.fuq.com
-www.eenadu.net
-m.magazineluiza.com.br
-www.google.co.uk
-faq.whatsapp.com
-in.indeed.com
-moviesda9.co
-mail.yahoo.com
-t.me
-m.tabor.ru
-news.google.com
-m.gsmarena.com
-onlyfans.com
-www.bilibili.tv
-wl2.peachavocado.com
-portal.auone.jp
-unifiedportal-mem.epfindia.gov.in
-uidai.gov.in
-www.flashscore.pl
-www.google.com.br
-m.beboo.ru
-login.yahoo.com
-tubidy.cool
-mail.ru
-www.etsy.com
-www.mediafire.com
-www.ilmeteo.it
-pl.pornhub.com
-transit.yahoo.co.jp
-chat.openai.com
-newtoki329.com
-www.cronista.com
-psiphon.news
-member.rakuten-sec.co.jp
-g.laroza.net
-skipthegames.com
-apnaludo.com
-www.xvideos53.com
-www.nu.nl
-www.sozcu.com.tr
-www.pussyspace.com
-m.pantip.com
-www.deviantart.com
-www.promiedos.com.ar
-www.homedepot.com
-www.tvguia.es
-www.wildberries.ru
-wol.jw.org
-rezka.ag
-www.macys.com
-www.buzzfeed.com
-www.dropbox.com
-hi.xhamster42.desi
-www.adobe.com
-www.coolmathgames.com
-rule34video.com
-www.google.co.jp
-cookpad.com
-ser.brstej.com
-togetter.com
-animeyuzu.com
-www.cityheaven.net
-www.furaffinity.net
-www.nikkei.com
-game.hiroba.dpoint.docomo.ne.jp
-www.argos.co.uk
-www.accuweather.com
-tw.yahoo.com
-br.xhamster.com
-www.ahindian.com
-liquipedia.net
-onedrive.live.com
-tathya.uidai.gov.in
-www.idealista.it
-www.pixiv.net
-testing.illuminateed.com
-uranai.nosv.org
-wl5.peachavocado.com
-hentailib.me
-www.chip.de
-www.redfin.com
-hot.noodlemagazine.com
-boards.4chan.org
-ru.sex-studentki.cash
-tukif.com
-kakuyomu.jp
-topics.smt.docomo.ne.jp
-jut.su
-www.xvideos.es
-www.zalando.de
-aniwatchtv.to
-uchi.ru
-www.fsiblog3.club
-www.yahoo.co.jp
-xnxx.com.se
-www.prokerala.com
-scratch.mit.edu
-www.emag.ro
-www.hdtube.porn
-meet.google.com
-m.search.naver.com
-www.bbc.co.uk
-xxxsexjav.com
-ficbook.net
-www.qorno.com
-github.com
-www.ikea.com
-www.target.com
-www.news.com.au
-www.haberler.com
-archive.org
-fr.m.wikipedia.org
-news.yahoo.co.jp
-mechacomic.jp
-www.terabox.app
-www.5movierulz.blog
-www.whatsapp.com
-cuetug.ntaonline.in
-www.walmart.com
-mangalib.me
-www.cnn.com
-xhamster2.com
-m.dcinside.com
-www.lemonde.fr
-www.universal-credit.service.gov.uk
-www.ad.nl
-www.annunci69.it
-m.ruliweb.com
-www.livescore.in
-www.xvideos4.com
-beta.character.ai
-s.dto.jp
-xxx18hot.com
-es.xhamster.com
-www.pornid.name
-www.moneycontrol.com
-av19.gg
-www3.nhk.or.jp
-news.goo.ne.jp
-www.sexybluefilm.com
-in.bookmyshow.com
-www.autotrader.co.uk
-www.telegraaf.nl
-www.newsweek.com
-www.sahibinden.com
-www.myntra.com
-gemini.google.com
-girlschannel.net
-hivemapper.com
-mat6tube.com
-ge.globo.com
-brainly.in
-de.pornhub.com
-www.filmweb.pl
-homepage.vivo.com
-diziwatch.net
-imhentai.xxx
-www.paypal.com
-www.kompas.com
-syosetu.org
-www.apple.com
-supjav.com
-www.kohls.com
-toyhou.se
-lectormanga.lat
-www.francetvinfo.fr
-www.xnxx2.com
-bet.caliente.mx
-bbs.animanch.com
-quillbot.com
-www.blic.rs
-www.digikala.com
-www.thscore.mobi
-ads.google.com
-vegamovies.ong
-www.gov.uk
-www.chess.com
-www.onet.pl
-shopee.com.br
-www.olx.com.br
-www.ixl.com
-www.ixbt.com
-www.ncbi.nlm.nih.gov
-dmarket.docomo.ne.jp
-articulo.mercadolibre.com.ar
-m.facebook.com
-sp.jra.jp
-xosodaiphat.com
-www.rezultati.com
-rt.pornhub.com
-www.52shuku.vip
-www.colamanga.com
-www.youporn.com
-motherless.com
-www.betway.co.za
-fapello.com
-kkstories.com
-mbasic.facebook.com
-fatalmodel.com
-www.hltv.org
-m.imdb.com
-www.bilibili.com
-www.nike.com
-www.amazon.com.br
-www.isaimini.com.sb
-www.pornhat.one
-www.milliyet.com.tr
-pfms.nic.in
-play.google.com
-www.livehindustan.com
-www.wannonce.com
-portal.librus.pl
-xhamster1.desi
-luxmovies.biz
-ru.m.wikipedia.org
-dpboss.services
-www.sexvid.pro
-xnxxcom.xyz
-www.croxyproxy.com
-pet.ibomma.bond
-www.xvideos.com
-drive.google.com
-pl.escort.club
-yaoilib.me
-znanija.com
-free.facebook.com
-wl1.peachavocado.com
-www.shutterstock.com
-www.eporner.com
-www2.hm.com
-newtoki328.com
-woman.excite.co.jp
-www.xvideos5.com
-br.search.yahoo.com
-spankbang.com
-www.gismeteo.ru
-www.uol.com.br
-fetlife.com
-serviziweb2.inps.it
-aternos.org
-wiadomosci.wp.pl
-www.speedtest.net
-gamewith.jp
-www.tuazar.com
-it.pornhub.com
-web.facebook.com
-www.erome.com
-www.screener.in
-client--gws-wiz-serp22-v1ddlcxru01-ioin.a-0.xyz
-services.gst.gov.in
-snapinsta.app
-www.xvideos2.com
-tw.news.yahoo.com
-one.google.com
-classroom.google.com
-happymod.com
-satta-king-fast.com
-remaker.ai
-www.ecosia.org
-video.hnext.jp
-pt.m.wikipedia.org
-open.spotify.com
-translate.google.com
-nhentai.net
-ww1.goojara.to
-m.site777.jp
-www.upwork.com
-www.terra.com.br
-www.sanook.com
-wapbold.net
-www.xnxx.com
-www.amazon.co.uk
-www.njuskalo.hr
-www.nairaland.com
-s.tabelog.com
-www.tbshop-ma.net
-m.cricbuzz.com
-www.amazon.ca
-www.expressen.se
-www.kleinanzeigen.de
-xxxhindi.to
-heylink.me
-www.dlsite.com
-www.hibiny.ru
-www.corriere.it
-www.nytimes.com
-xnxxsex.net
-www.ups.com
-rx.sp.mbga.jp
-brainly.com.br
-www.hepsiburada.com
-www.tripadvisor.com
-sportowefakty.wp.pl
-www.indianbfvideos.com
-82xnxx.com
-m.daum.net
-manhuascan.io
-www.mercadolibre.com.mx
-spankbang.party
-mangatoto.com
-en.wikipedia.org
-www.amazon.it
-persona.patria.org.ve
-www.tamildhool.net
-zinmanga.com
-www.otto.de
-91club06.com
-www.pornpics.com
-suumo.jp
-www.boatrace.jp
-www.infobae.com
-m.inven.co.kr
-quizlet.com
-turbo.az
-www.y2mate.com
-www.toppr.com
-studio.youtube.com
-m.avito.ru
-blog.livedoor.jp
-m.fabguys.com
-www.microsoft.com
-www.bol.com
-www.nitrotype.com
-www.temu.com
-goal.co
-www.khabaronline.ir
-91club.club
-allegro.pl
-funnyjunk.com
-auctions.yahoo.co.jp
-pt.pornhub.com
-lermanga.org
-rajshaladarpan.nic.in
-wto.to
-m.scan-manga.com
-www.canva.com
-play.contents.plus-msg.auone.jp
-ja.m.wikipedia.org
-www.carsensor.net
-www.nettruyenee.com
-zoom.us
-www.sarkariresult.com
-page.auctions.yahoo.co.jp
-shopee.co.id
-www.nesine.com
-www.wnacg.com
-www.dafont.com
-www.theguardian.com
-www.sinsay.com
-a24.app.gree-pf.net
-roboguru.ruangguru.com
-www.studocu.com
-www.timesbull.com
-yatv.pro
-nouen.sp.mbga.jp
-cardgames.io
-sp.pf.mbga.jp
-passbook.epfindia.gov.in
-pg.access-33.com
-wonporn.com
-yandex.com
-noutore.enjoy.point.auone.jp
-www.primevideo.com
-articulo.mercadolibre.com.mx
-www.pornhub.com
-www.rightmove.co.uk
-es.scribd.com
-jp.pornhub.com
-www.repubblica.it
-id.m.wikipedia.org
-m.skelbiu.lt
-www.marktplaats.nl
-es.m.wikipedia.org
-search.yahoo.com
-jp.spankbang.com

From ac820d163a6f8d220cc1c61d7f030045867545d4 Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:50:41 -0700
Subject: [PATCH 22/23] moved back to 100 domains

---
 testing/domains.csv | 400 --------------------------------------------
 1 file changed, 400 deletions(-)

diff --git a/testing/domains.csv b/testing/domains.csv
index 741b4200..c14045c1 100644
--- a/testing/domains.csv
+++ b/testing/domains.csv
@@ -98,403 +98,3 @@ giris.turkiye.gov.tr
 www.protothema.gr
 18comic.vip
 m.olx.pl
-tvn24.pl
-m.naver.com
-www.cardmarket.com
-wordwall.net
-www.varzesh3.com
-www.mynet.com
-es.aliexpress.com
-www.hentai.name
-www.hdhindisex.com
-www.netshoes.com.br
-wl4.peachavocado.com
-hackerdoslot.com
-www.hln.be
-xhdepot.site
-xvideosporno.blog.br
-czbooks.net
-genius.com
-ekstrabladet.dk
-launchpad.classlink.com
-danbooru.donmai.us
-www.action.com
-onlar.az
-doublelist.com
-ja.wikipedia.org
-news.livedoor.com
-www.anses.gob.ar
-mlbpark.donga.com
-support.google.com
-www.kidsa-z.com
-www.merkur.de
-accounts.spotify.com
-index.hu
-bollyflix.nexus
-www.hindustantimes.com
-web.spaggiari.eu
-scholar.google.com
-www.bestbuy.com
-www.quora.com
-www.instagram.com
-www.avito.ru
-www.sportskeeda.com
-www.xnxx.tv
-g1.globo.com
-www.daily.co.jp
-play.blooket.com
-simpcity.su
-www.kamababa.desi
-www.starfall.com
-nz.ua
-saednews.com
-www.finn.no
-member.sss.gov.ph
-hianime.to
-www.mangago.me
-e621.net
-ar.m.wikipedia.org
-gdz.ru
-byjus.com
-www.viva.co.id
-www.vinted.fr
-www.google.com.tr
-www.betpawa.ug
-www.sciencedirect.com
-brainly.co.id
-www.amazon.eg
-www.mercadolivre.com.br
-www.google.com.hk
-www.indeed.com
-www.lanacion.com.ar
-nl.pornhub.com
-steamcommunity.com
-an1.com
-mangakakalot.com
-truyenfull.vn
-linktr.ee
-m.cafe.daum.net
-vnexpress.net
-www.tgju.org
-www.ukr.net
-betproexch.com
-www.spotify.com
-fr.xhamster.com
-www.sugotoku.docomo.ne.jp
-kolkataff.in
-www.patria.org.ve
-www.roblox.com
-bdvenlinea.banvenez.com
-mail.google.com
-www.google.de
-news.infoseek.co.jp
-www.amazon.es
-www.espncricinfo.com
-zh.m.wikipedia.org
-www.aajtak.in
-n.news.naver.com
-www.livescore.com
-www.remove.bg
-fa.m.wikipedia.org
-secure.livechatinc.com
-www.almasryalyoum.com
-www.ebay.co.uk
-synergia.librus.pl
-www.ninisite.com
-www.google.pl
-chromewebstore.google.com
-m.olx.ua
-s.kakaku.com
-wimt.confirmtkt.com
-www.globo.com
-www.disneyplus.com
-es.pornhub.com
-gelbooru.com
-servicioscorp.anses.gob.ar
-www.xvideos3.com
-www.irs.gov
-xbato.com
-salonboard.com
-tel.ibomma.nexus
-tw.stock.yahoo.com
-www.tjk.org
-newtoki331.com
-readmanga.live
-www.programme-tv.net
-br.pinterest.com
-www.keralalotterytoday.com
-xxxbfvideos.org
-escortalligator.com.listcrawler.eu
-www.jutarnji.hr
-www.namasha.com
-finance.yahoo.com
-missav.com
-www.joyclub.de
-www.ign.com
-www.amazon.in
-www.independent.co.uk
-www.ryanair.com
-mangaschan.net
-mangatoto.org
-www.goojara.to
-www.tradingview.com
-www.twitch.tv
-11xplay.pro
-elpais.com
-snaptik.app
-people.com
-m.humoruniv.com
-ru.xhamster.com
-xhwide4.com
-www.inven.co.kr
-ssstik.io
-sso.acesso.gov.br
-www.cdiscount.com
-www.betika.com
-enabiz.gov.tr
-sattasport.in
-xoso.com.vn
-xnxx.health
-www.ynet.co.il
-www.google.com
-sattamatkano1.me
-hanime1.me
-novel18.syosetu.com
-www.dailymail.co.uk
-www.lefigaro.fr
-www.casinoplus.com.ph
-www.slideshare.net
-e-hentai.org
-m.tribunnews.com
-www.jlpt-maja.org.mm
-olxmewah.com
-www.futbin.com
-www.lxtgame.com
-www.wattpad.com
-music-fa.com
-www.iltalehti.fi
-www.google.com.mx
-timesofindia.indiatimes.com
-www.youjizz.com
-fantia.jp
-tools.usps.com
-www.aemet.es
-www.boyfriendtv.com
-jable.tv
-www.24h.com.vn
-faphouse.com
-www.list.am
-hot-sex-tube.com
-www.hurriyet.com.tr
-noodlemagazine.com
-www.xnxx3.com
-satta-king-fixed-no.in
-calendar.google.com
-www.3bmeteo.com
-coomer.su
-m.hqporner.com
-www.apornvideo.com
-dewafilm.xyz
-smt.docomo.ne.jp
-twitter.com
-account.samsung.com
-archiveofourown.org
-vlxx.moe
-m.sexemodel.com
-www.imagefap.com
-www.ojogodobicho.com
-www.dogdrip.net
-www.google.it
-finance.yahoo.co.jp
-www.aparat.com
-moi.ibomma.re
-pl.xhamster.com
-www.idnes.cz
-xhamster42.desi
-kompoz2.com
-tel.ibomma.rsvp
-m.vk.com
-www.amazon.fr
-www.cnnindonesia.com
-en.luxuretv.com
-select.mamastar.jp
-pubmed.ncbi.nlm.nih.gov
-www.biblegateway.com
-www.marmiton.org
-dantri.com.vn
-s.mogg.ameba.jp
-www.nikkansports.com
-v.daum.net
-www.avto.net
-www.naver.com
-m.hollywoodbets.net
-m.ok.ru
-www.ndtv.com
-rajwap.cc
-search.yahoo.co.jp
-ya.ru
-www.foxnews.com
-www.marca.com
-www.youtube.com
-www.bing.com
-xxxbfvideos.net
-www.nettruyenff.com
-zistop.com
-zbporn.tv
-www.unotv.com
-www.amazon.com
-sxyprn.com
-theqoo.net
-khoahoc.vietjack.com
-namu.wiki
-www.fulldesisex.com
-schools.emaktab.uz
-outlook.live.com
-www.discogs.com
-de.pornhub.org
-apps.facebook.com
-old.reddit.com
-www.flashscore.fr
-www.leroymerlin.fr
-www.jw.org
-www.aliexpress.com
-www.scribd.com
-bokepindoh.sbs
-duckduckgo.com
-drive.usercontent.google.com
-ecnavi.jp
-nypost.com
-www.amazon.co.jp
-www.ladepeche.fr
-www.patreon.com
-xhamster.desi
-game.granbluefantasy.jp
-www.google.co.in
-tsn.ua
-www.sexvid.xxx
-udn.com
-la.spankbang.com
-ge.xhamster.desi
-tamilyogi.red
-telegram.org
-www.otomoto.pl
-www.cnbcindonesia.com
-rule34.xxx
-www.bpexch.com
-topmanhua.com
-www.imdb.com
-xhtab3.com
-m.haber7.com
-porno18.site
-www.wp.pl
-stackoverflow.com
-www.sikayetvar.com
-actu.fr
-item.rakuten.co.jp
-produto.mercadolivre.com.br
-litnet.com
-grapee.jp
-myapps.classlink.com
-www.agoda.com
-www.index.hr
-beauty.hotpepper.jp
-mbigpara.hurriyet.com.tr
-kemono.su
-www.staseraintv.com
-shinigami.cx
-www.ouest-france.fr
-direct.lc.chat
-m.shein.com
-www.vg.no
-www.w3schools.com
-esaj.tjsp.jus.br
-hiroba.dpoint.docomo.ne.jp
-www.amazon.com.tr
-www.ozon.ru
-lichess.org
-www.goodreads.com
-www.mathplayground.com
-www.nexusmods.com
-app.hubspot.com
-funpay.com
-m.kooora.com
-www.wordreference.com
-www.goal.com
-truyenqqvn.com
-uk.indeed.com
-www.fanpage.it
-www.lowes.com
-www.diretta.it
-search.naver.com
-www.leboncoin.fr
-m.ppomppu.co.kr
-www.dns-shop.ru
-www.metropoles.com
-chaturbate.com
-www.serviporno.com
-bato.to
-game8.jp
-alarbda.com
-xxxvideo.link
-www.pinterest.com
-www.vietjack.com
-ita.xhamster.com
-www.trendyol.com
-www.ebay.it
-vegamovies.rsvp
-www.forbes.com
-return.gst.gov.in
-www.jcpenney.com
-xn--l3cg7a8a0cwa3f.com
-www.realtor.com
-ameblo.jp
-iporntv.net
-www.google.fr
-chance.enjoy.point.auone.jp
-business.facebook.com
-m.chaturbate.com
-www.5movierulz.dog
-id.xhamster.com
-weather.com
-www.chegg.com
-www.ixxx.com
-www.sportybet.com
-auto.ria.com
-m.youm7.com
-www.amazon.de
-mpokec.azet.sk
-www.filmyzilla.com.ng
-www.jra.go.jp
-in3x.net
-www.bt.dk
-www.thesun.co.uk
-it.m.wikipedia.org
-manhwaclan.com
-screenrant.com
-trilltrill.jp
-www.jiocinema.com
-www.tvonenews.com
-login.caixa.gov.br
-finance.naver.com
-www.bakecaincontrii.com
-emaktab.uz
-movie.eroterest.net
-www.1mg.com
-www.eltiempo.es
-tr.m.wikipedia.org
-tmohentai.com
-www.seznam.cz
-auth.uber.com
-photos.google.com
-www.flashscore.com.br
-kuji.dmarket.docomo.ne.jp
-docs.google.com
-www.amazon.com.mx
-sites.google.com
-cricbet99.win
-www.ilovepdf.com
-www.skroutz.gr
-my.porno365.pics
-support.microsoft.com
-freeip.me
-www.turkiye.gov.tr

From bed56ec450f862ea04cc9205fd4c7bad0ce27e5c Mon Sep 17 00:00:00 2001
From: phillip-stephens <phillip@cs.stanford.edu>
Date: Tue, 28 May 2024 15:54:08 -0700
Subject: [PATCH 23/23] add comments explaining integration tests

---
 .github/workflows/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7ceaa417..111008ce 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,6 +20,8 @@ jobs:
           else
             echo 'All .go files in repo have an appropriate header'
           fi
+  # This tests runs both unit tests and integration tests that validate that ZDNS can pull many DNS record types against
+  # our controlled domain
   build-and-test:
     runs-on: ubuntu-latest
     steps:
@@ -41,6 +43,7 @@ jobs:
           sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
           python --version
           ./testing/integration_tests.py
+  # This test runs an integration test for a ZDNS scan on 100 domains to validate that the A records are correct
   build-and-test-large-scale:
     runs-on: ubuntu-latest
     steps: