-
Notifications
You must be signed in to change notification settings - Fork 451
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: mashable parser As usual the date is giving me issues because of formatting discrepancies: AssertionError: '2016-12-13T22:33:06.000Z' == '2016-12-14T03:33:06.000Z' Not sure how we wanna deal with Twitter card embeds that don’t show up? Also, image credits did not show up in preview. * test: fixed date format * transforming .image-credit to figcaption
- Loading branch information
1 parent
cddc1af
commit dec49ab
Showing
4 changed files
with
150 additions
and
0 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
export const MashableComExtractor = { | ||
domain: 'mashable.com', | ||
|
||
title: { | ||
selectors: [ | ||
'h1.title', | ||
], | ||
}, | ||
|
||
author: { | ||
selectors: [ | ||
'span.author_name a', | ||
], | ||
}, | ||
|
||
date_published: { | ||
selectors: [ | ||
['meta[name="og:article:published_time"]', 'value'], | ||
], | ||
}, | ||
|
||
lead_image_url: { | ||
selectors: [ | ||
['meta[name="og:image"]', 'value'], | ||
], | ||
}, | ||
|
||
content: { | ||
selectors: [ | ||
'section.article-content.blueprint', | ||
], | ||
|
||
// Is there anything in the content you selected that needs transformed | ||
// before it's consumable content? E.g., unusual lazy loaded images | ||
transforms: { | ||
'.image-credit': 'figcaption', | ||
}, | ||
|
||
// Is there anything that is in the result that shouldn't be? | ||
// The clean selectors will remove anything that matches from | ||
// the result | ||
clean: [ | ||
|
||
], | ||
}, | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
import assert from 'assert'; | ||
import fs from 'fs'; | ||
import URL from 'url'; | ||
import cheerio from 'cheerio'; | ||
|
||
import Mercury from 'mercury'; | ||
import getExtractor from 'extractors/get-extractor'; | ||
import { excerptContent } from 'utils/text'; | ||
|
||
describe('MashableComExtractor', () => { | ||
describe('initial test case', () => { | ||
let result; | ||
let url; | ||
beforeAll(() => { | ||
url = | ||
'http://mashable.com/2016/12/13/mysterious-plane-flying-over-new-york/?utm_cid=hp-n-1#sxBI1HiPKsqG'; | ||
const html = | ||
fs.readFileSync('./fixtures/mashable.com/1481670648585.html'); | ||
result = | ||
Mercury.parse(url, html, { fallback: false }); | ||
}); | ||
|
||
it('is selected properly', () => { | ||
// This test should be passing by default. | ||
// It sanity checks that the correct parser | ||
// is being selected for URLs from this domain | ||
const extractor = getExtractor(url); | ||
assert.equal(extractor.domain, URL.parse(url).hostname); | ||
}); | ||
|
||
it('returns the title', async () => { | ||
// To pass this test, fill out the title selector | ||
// in ./src/extractors/custom/mashable.com/index.js. | ||
const { title } = await result; | ||
|
||
// Update these values with the expected values from | ||
// the article. | ||
assert.equal(title, 'Mysterious plane circling Manhattan sparks concern and intrigue'); | ||
}); | ||
|
||
it('returns the author', async () => { | ||
// To pass this test, fill out the author selector | ||
// in ./src/extractors/custom/mashable.com/index.js. | ||
const { author } = await result; | ||
|
||
// Update these values with the expected values from | ||
// the article. | ||
assert.equal(author, 'Nicole Gallucci'); | ||
}); | ||
|
||
it('returns the date_published', async () => { | ||
// To pass this test, fill out the date_published selector | ||
// in ./src/extractors/custom/mashable.com/index.js. | ||
const { date_published } = await result; | ||
|
||
// Update these values with the expected values from | ||
// the article. | ||
assert.equal(date_published, '2016-12-13T22:33:06.000Z'); | ||
}); | ||
|
||
it('returns the lead_image_url', async () => { | ||
// To pass this test, fill out the lead_image_url selector | ||
// in ./src/extractors/custom/mashable.com/index.js. | ||
const { lead_image_url } = await result; | ||
|
||
// Update these values with the expected values from | ||
// the article. | ||
assert.equal(lead_image_url, 'http://a.amz.mshcdn.com/media/ZgkyMDE2LzEyLzEzL2UxL2xpbGlzYW1zbWFzaGFibGU1XzcyMC4wMWZkOS5qcGcKcAl0aHVtYgkxMjAweDYzMAplCWpwZw/29e123a7/0e0/lili-sams-mashable-5_720.jpg'); | ||
}); | ||
|
||
it('returns the content', async () => { | ||
// To pass this test, fill out the content selector | ||
// in ./src/extractors/custom/mashable.com/index.js. | ||
// You may also want to make use of the clean and transform | ||
// options. | ||
const { content } = await result; | ||
|
||
const $ = cheerio.load(content || ''); | ||
|
||
const first13 = excerptContent($('*').first().text(), 13); | ||
|
||
// Update these values with the expected values from | ||
// the article. | ||
assert.equal(first13, 'A large military-style plane, which looked remarkably like a C-130, circled Manhattan for'); | ||
}); | ||
}); | ||
}); |