fix: navigation timeout when generating test case for raw story webpage

packages/readabilityjs/package.json

@@ -32,10 +32,10 @@
     "linkedom": "^0.14.9",
     "mocha": "^8.2.0",
     "nock": "^13.3.1",
-    "puppeteer-core": "^22.8.0",
-    "puppeteer-extra": "^3.3.4",
-    "puppeteer-extra-plugin-adblocker": "^2.13.5",
-    "puppeteer-extra-plugin-stealth": "^2.11.1",
+    "puppeteer-core": "^22.12.1",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-adblocker": "^2.13.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
     "sinon": "^7.3.2"
   },
   "dependencies": {

packages/readabilityjs/test/generate-testcase.js

@@ -108,27 +108,25 @@ async function fetchSource(url, callbackFn) {
 
   const browser = await puppeteer.launch({
     args: [
-      '--allow-running-insecure-content',
       '--autoplay-policy=user-gesture-required',
       '--disable-component-update',
       '--disable-domain-reliability',
-      '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process',
       '--disable-print-preview',
       '--disable-setuid-sandbox',
-      '--disable-site-isolation-trials',
       '--disable-speech-api',
-      '--disable-web-security',
-      '--disk-cache-size=33554432',
       '--enable-features=SharedArrayBuffer',
       '--hide-scrollbars',
-      '--disable-gpu',
       '--mute-audio',
       '--no-default-browser-check',
       '--no-pings',
       '--no-sandbox',
       '--no-zygote',
-      '--window-size=1920,1080',
       '--disable-extensions',
+      '--disable-dev-shm-usage',
+      '--no-first-run',
+      '--disable-background-networking',
+      '--disable-gpu',
+      '--disable-software-rasterizer',
     ],
     defaultViewport: {
       deviceScaleFactor: 1,
@@ -138,15 +136,18 @@ async function fetchSource(url, callbackFn) {
       isMobile: false,
       width: 1920,
     },
-    headless: true,
+    headless: 'shell',
+    dumpio: true, // show console logs in the terminal
     executablePath: process.env.CHROMIUM_PATH || '/opt/homebrew/bin/chromium',
+    // filter out targets
+    targetFilter: (target) =>
+      target.type() !== 'other' || !!target.url(),
   })
 
   const page = await browser.newPage()
   if (!enableJavascriptForUrl(url)) {
     await page.setJavaScriptEnabled(false)
   }
-  await page.setUserAgent(userAgentForUrl(url))
 
   try {
     /*
@@ -155,18 +156,29 @@ async function fetchSource(url, callbackFn) {
      * mathjax content when present.
      */
     await page.setRequestInterception(true)
+
+    let requestCount = 0
     page.on('request', (request) => {
-      if (
-        request.resourceType() === 'script' &&
-        request.url().toLowerCase().indexOf('mathjax') > -1
-      ) {
-        request.abort()
-      } else {
-        request.continue()
-      }
+      ;(async () => {
+        if (request.resourceType() === 'font') {
+          // Disallow fonts from loading
+          return request.abort()
+        }
+        if (requestCount++ > 100) {
+          return request.abort()
+        }
+        if (
+          request.resourceType() === 'script' &&
+          request.url().toLowerCase().indexOf('mathjax') > -1
+        ) {
+          return request.abort()
+        }
+
+        await request.continue()
+      })()
     })
 
-    await page.goto(url, { waitUntil: ['networkidle2'] })
+    await page.goto(url, { waitUntil: ['networkidle0'] })
 
     /* scroll with a 5 second timeout */
     await Promise.race([

packages/readabilityjs/test/test-pages/raw-story/distiller.html

@@ -0,0 +1,33 @@
+<div><p>
+                                                                            Former President <a href="https://www.rawstory.com/trump-musk/" target="_blank">Donald Trump&#39;s</a> rambling interview with <a href="https://www.rawstory.com/elon-musk-trump-2668952236/" target="_blank">Elon Musk</a> touched on climate change, border security, foreign dictators and inflation.
+                                                                        </p><p>
+                                                                                But it was his apparent appreciation of &#34;beautiful&#34; <a href="https://www.rawstory.com/kamala-harris/">Kamala Harris</a> — and his comparison to his wife, Melania <a href="https://www.rawstory.com/trump-news/">Trump</a> — that left the internet with raised eyebrows.
+                                                                            </p><p>
+                                                                                During Trump&#39;s <a href="https://www.rawstory.com/elon-musk-trump-2668951495/" target="_blank">interview on X with tech billionaire Musk</a>, the MAGA leader said his Democratic rival has received a &#34;free ride&#34; from the media so far.
+                                                                            </p><p>
+                                                                                &#34;But I saw a picture of her on Time magazine today, she looks like the most beautiful actress ever to live,&#34; Trump said. &#34;It was a drawing. And actually she looked very much like a great first lady: Melania. She didn&#39;t look like Camilla. That&#39;s right. But of course she&#39;s a beautiful woman. So we&#39;ll leave it at that.&#34;
+                                                                            </p><p>
+                                                                                The comment left the internet collectively cringed at the comment, including journalist Aaron Rupar, who called the recording &#34;<a href="https://x.com/atrupar/status/1823183367380034024" target="_blank">weird stuff</a>!&#34;
+                                                                            </p><p>
+                                                                                &#34;I KNEW IT. His <a href="https://x.com/AshaRangappa_/status/1823182857533337645" target="_blank">brain is short-circuiting</a> because she is attractive AND formidable. lol he is toast,&#34; chided former FBI special agent and legal contributer Asha Rangappa.
+                                                                            </p><p>
+                                                                                She added that Trump &#34;recognizes and is slightly scared of — and in a weird way, craves the approval of — strong, intelligent women,&#34; such as his former Democratic opponent Hillary Clinton and former House Speaker <a href="https://www.rawstory.com/tag/nancy-pelosi">Nancy Pelosi</a>.
+                                                                            </p><p>
+                                                                                &#34;Some Mommy issues going on there,&#34; she added, noting that Trump equates attractive women with &#34;&#39;<a href="https://x.com/AshaRangappa_/status/1823183719148171361" target="_blank">dumb&#39; or weak</a>,&#34; so he can &#34;sexualize and exploit and dismiss.&#34;
+                                                                            </p><p>
+                                                                                &#34;This is the point of the call where I say, &#39;Well, it&#39;s been <a href="https://x.com/kelsientaggart/status/1823186094495846647" target="_blank">great catching up grandpa</a>, but I&#39;ve got to go,&#39; quipped Kelsie Taggart, digital media director at the left-leaning American Bridge PAC.
+                                                                            </p><p>
+                                                                                &#34;Donald Trump mentions this Time magazine cover, says Kamala Harris looks like an actress in the illustration and then says she looks like … his wife, <a href="https://x.com/ccadelago/status/1823182901875449929" target="_blank">Melania Trump</a>. ???&#34; questioned Christopher Cadelago, Politico&#39;s <a href="https://www.rawstory.com/tag/california">California</a> bureau chief, wrote on X, attaching a photo of the image.
+                                                                            </p><p>
+                                                                                &#34;<a href="https://x.com/RpsAgainstTrump/status/1823189790864376209" target="_blank">What??</a>&#34; questioned the Never-Trump account Republicans against Trump.
+                                                                            </p><p>
+                                                                                &#34;Trump basically just said the drawing of Kamala on time magazine was <a href="https://x.com/JessicaLBurbank/status/1823182494440816787" target="_blank">hot and compared it to his wife Melania</a> lol,&#34; laughed Jessica Burbank, of the More Perfect Union podcast.
+                                                                            </p><div>
+                                                                        <span>CONTINUE READING</span><span>Show less</span>
+                                                                    </div><p>
+                                                                            A 72-year-old Colorado woman who stormed the Capitol on <a href="https://www.rawstory.com/trump-mini-trial/" target="_blank">January 6, 2021</a>, will avoid jail time after a judge sentenced her to a year of probation and a $103,000 fine Monday, <a href="https://coloradosun.com/2024/08/12/rebecca-lavrenz-praying-grandma-jan-6-sentenc/" target="_blank">the Colorado Sun reports.</a>
+                                                                        </p><p>
+                                                                                Rebecca Lavrenz, who took on the moniker “<a href="https://www.rawstory.com/trump-accused-of-invoking-jan-6/" target="_blank">J6</a> Praying Grandma,” was convicted on four misdemeanor counts for entering the Capitol and disorderly conduct for participating in the insurrection, according to the Sun. Prosecutors initially pushed for Lavrenz to serve up to 10 months in prison for continuing to defend the insurrection, writing in a recent filing that “her unrepentant promotion of the riot is powerful evidence that she continues to pose a threat to future acts of political violence.”
+                                                                            </p><p>
+                                                                                The Colorado Sun noted that Lavrenz has <a href="https://www.givesendgo.com/rebeccalavrenz" target="_blank">raised</a> <a href="https://www.givesendgo.com/rebeccalavrenzJ6" target="_blank">more than</a> $230,000 via online fundraisers by promoting her participation in the riot.
+                                                                            </p></div>

packages/readabilityjs/test/test-pages/raw-story/expected-metadata.json

@@ -0,0 +1,12 @@
+{
+  "title": "'Interview crashed and burst into flames': Trump and Musk mocked amid stream's launch woes",
+  "byline": "Leigh Tauss",
+  "dir": null,
+  "excerpt": "The failure to launch of Elon Musk's planned audio live stream on X with former President Donald Trump quickly became the subject of online mockery on the very same social platform, including by the official campaign of Democratic presidential nominee Kamala Harris.\"BREAKING: Twitter,\" the pro-Harri...",
+  "siteName": "Raw Story - Celebrating 20 Years of Independent Journalism",
+  "siteIcon": "https://assets.rbl.ms/24986912/origin.png",
+  "previewImage": "https://www.rawstory.com/media-library/chief-executive-officer-of-spacex-and-tesla-and-owner-of-twitter-elon-musk-attends-the-viva-technology-conference-dedicated-to.jpg?id=53161798&width=1200&height=600&coordinates=0%2C0%2C0%2C104",
+  "publishedDate": "2024-08-13T01:09:23.000Z",
+  "language": "English",
+  "readerable": true
+}