Merge branch 'master' into feat-natgeo-extractor

src/extractors/custom/index.js

@@ -56,6 +56,7 @@ export * from './uproxx.com';
 export * from './www.eonline.com';
 export * from './www.miamiherald.com';
 export * from './www.refinery29.com';
+export * from './www.si.com';
 export * from './www.rawstory.com';
 export * from './www.cnet.com';
 export * from './www.cinemablend.com';

src/extractors/custom/www.si.com/index.js

@@ -0,0 +1,68 @@
+export const WwwSiComExtractor = {
+  domain: 'www.si.com',
+
+  title: {
+    selectors: [
+      'h1',
+      'h1.headline',
+    ],
+  },
+
+  author: {
+    selectors: [
+      ['meta[name="author"]', 'value'],
+    ],
+  },
+
+  date_published: {
+    selectors: [
+      '.timestamp',
+    ],
+
+    timezone: 'America/New_York',
+  },
+
+  dek: {
+    selectors: [
+      '.quick-hit ul',
+    ],
+  },
+
+  lead_image_url: {
+    selectors: [
+      ['meta[name="og:image"]', 'value'],
+    ],
+  },
+
+  content: {
+    selectors: [
+      ['p', '.marquee_large_2x', '.component.image'],
+    ],
+
+    // Is there anything in the content you selected that needs transformed
+    // before it's consumable content? E.g., unusual lazy loaded images
+    transforms: {
+
+      noscript: ($node) => {
+        const $children = $node.children();
+        if ($children.length === 1 && $children.get(0).tagName === 'img') {
+          return 'figure';
+        }
+
+        return null;
+      },
+    },
+
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: [
+      [
+        '.inline-thumb',
+        '.primary-message',
+        '.description',
+        '.instructions',
+      ],
+    ],
+  },
+};

src/extractors/custom/www.si.com/index.test.js

@@ -0,0 +1,97 @@
+import assert from 'assert';
+import fs from 'fs';
+import URL from 'url';
+import cheerio from 'cheerio';
+
+import Mercury from 'mercury';
+import getExtractor from 'extractors/get-extractor';
+import { excerptContent } from 'utils/text';
+
+describe('WwwSiComExtractor', () => {
+  describe('initial test case', () => {
+    let result;
+    let url;
+    beforeAll(() => {
+      url =
+        'http://www.si.com/nfl/2017/01/12/dallas-cowboys-stephen-jerry-jones';
+      const html =
+        fs.readFileSync('./fixtures/www.si.com/1484253704408.html');
+      result =
+        Mercury.parse(url, html, { fallback: false });
+    });
+
+    it('is selected properly', () => {
+      // This test should be passing by default.
+      // It sanity checks that the correct parser
+      // is being selected for URLs from this domain
+      const extractor = getExtractor(url);
+      assert.equal(extractor.domain, URL.parse(url).hostname);
+    });
+
+    it('returns the title', async () => {
+    // To pass this test, fill out the title selector
+    // in ./src/extractors/custom/www.si.com/index.js.
+      const { title } = await result;
+
+    // Update these values with the expected values from
+    // the article.
+      assert.equal(title, 'A Breath of Fresh Heir: Stephen Jones following in the shadow cast by his father');
+    });
+
+    it('returns the author', async () => {
+    // To pass this test, fill out the author selector
+    // in ./src/extractors/custom/www.si.com/index.js.
+      const { author } = await result;
+
+    // Update these values with the expected values from
+    // the article.
+      assert.equal(author, 'Austin Murphy');
+    });
+
+    it('returns the date_published', async () => {
+    // To pass this test, fill out the date_published selector
+    // in ./src/extractors/custom/www.si.com/index.js.
+      const { date_published } = await result;
+
+    // Update these values with the expected values from
+    // the article.
+      assert.equal(date_published, '2017-01-12T05:00:00.000Z');
+    });
+
+    it('returns the dek', async () => {
+    // To pass this test, fill out the dek selector
+    // in ./src/extractors/custom/www.si.com/index.js.
+      const { dek } = await result;
+
+    // Update these values with the expected values from
+    // the article.
+      assert.equal(dek, 'How did the Cowboys half two decades of mediocrity and land the top playff seed in the NFC? Start with the one man who will stand up to owner Jerry Jones: his son Stephen, the EVP, CEO and Director of Player Personnel of the Dallas Cowboys.');
+    });
+
+    it('returns the lead_image_url', async () => {
+    // To pass this test, fill out the lead_image_url selector
+    // in ./src/extractors/custom/www.si.com/index.js.
+      const { lead_image_url } = await result;
+
+    // Update these values with the expected values from
+    // the article.
+      assert.equal(lead_image_url, 'http://cdn-s3.si.com/styles/inline_gallery_desktop/s3/images/stephen-jerry-jones-dallas-cowboys.jpg?itok=tmQDYtjs');
+    });
+
+    it('returns the content', async () => {
+      // To pass this test, fill out the content selector
+      // in ./src/extractors/custom/www.si.com/index.js.
+      // You may also want to make use of the clean and transform
+      // options.
+      const { content } = await result;
+
+      const $ = cheerio.load(content || '');
+
+      const first13 = excerptContent($('*').first().text(), 13);
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(first13, 'Stephen Jones already wears three hats for the Cowboys, but on a cloudy');
+    });
+  });
+});