From fadd198d0462af8d334db8a95aa87e338a77fbc2 Mon Sep 17 00:00:00 2001 From: Janet Date: Thu, 2 Feb 2017 17:43:35 -0500 Subject: [PATCH] feat: abcnewsgo parser (#90) * feat: abcnewsgo parser --- fixtures/abcnews.go.com/1481922563840.html | 1 + src/extractors/custom/abcnews.go.com/index.js | 51 +++++++++++ .../custom/abcnews.go.com/index.test.js | 87 +++++++++++++++++++ src/extractors/custom/index.js | 1 + 4 files changed, 140 insertions(+) create mode 100644 fixtures/abcnews.go.com/1481922563840.html create mode 100644 src/extractors/custom/abcnews.go.com/index.js create mode 100644 src/extractors/custom/abcnews.go.com/index.test.js diff --git a/fixtures/abcnews.go.com/1481922563840.html b/fixtures/abcnews.go.com/1481922563840.html new file mode 100644 index 000000000..79e814008 --- /dev/null +++ b/fixtures/abcnews.go.com/1481922563840.html @@ -0,0 +1 @@ + Hillary Clinton: Putin's Alleged Involvement in Democratic Hack Stems From Longtime Grudge - ABC News

Hillary Clinton: Putin's Alleged Involvement in Democratic Hack Stems From Longtime Grudge

Dec 16, 2016, 12:37 PM ET
PHOTO: Hillary Clinton speaks during a ceremony to unveil a portrait of Senate Minority Leader Harry Reid, Dec. 8, 2016, in Washington.PlayEvan Vucci/AP Photo
WATCH Clinton: Putin's Alleged Involvement in Hack Stems From Grudge

Hillary Clinton has an explanation for Vladimir Putin's alleged involvement in the hacking of Democratic organizations during the 2016 presidential election: The Russian leader has a longtime grudge against her, dating back to 2011 when she was secretary of state and made comments that were critical of Russia's parliamentary elections.

Two Clinton donors tell ABC News that Clinton made the claim Thursday night at Manhattan's Plaza Hotel during an event dubbed a "thank you" party, at which she expressed her gratitude to donors. Her former running mate Tim Kaine was in attendance, as was New York governor Andrew Cuomo.

The donors also say there is growing anger in the Clinton camp toward the FBI, saying they played a monumental role in allegedly handing the election to Donald Trump, in regards to FBI Director James Comey's handling of Clinton's email scandal.

Clinton campaign chairman John Podesta wrote an op-ed in the Washington Post Thursday with the headline "Something is deeply broken at the FBI," calling the department out for what he said was a "lackadaisical response to the very real Russian plot to subvert a national election."

At Thursday's event, some attendees were spotted crying while Clinton spoke. Looking upbeat and greeting guests, she encouraged attendees to help elect Democrats to office and stay involved in the party's future.

The Basis Behind the Alleged Grudge

There is a widespread perception among followers of U.S.-Russia relations that Putin holds a grudge against Clinton over what he sees as her efforts to have him pushed out when she was secretary of state.

In 2011, massive street demonstrations broke out in Moscow protesting the blatant rigging of Russia’s parliamentary elections in favor of Putin’s party.

The demonstrations, which saw as many as 100,000 people march, morphed into a larger protest against Putin and became the most serious popular threat to his presidency he has ever faced.

Putin blamed Clinton for the protests, accusing her of inciting them and saying she immediately tried to egg them on.

"I looked at the first reaction of our U.S. partners," Putin said at the time. "The first thing that the secretary of state did was say that they [the elections] were not honest and not fair, but she had not even yet received the material from the observers.

"She set the tone for some actors in our country and gave them a signal," Putin continued. "They heard the signal and with the support of the U.S. State Department began active work."

Political Implications

The Kremlin believes that the U.S. has seized on demonstrations in former Soviet countries and elsewhere to try to effect regime change in the name of Western democratic values, followers of U.S.-Russia relations say.

It is widely believed that Putin thinks that Clinton and her State Department were out to get him and Putin has some basis for this: In 2008, he stepped down as president for a term as a means of dodging a constitutional limit on consecutive terms, placing his longtime collaborator Dmitry Medvedev in the presidency while he kept much of the real power as prime minister.

Nonetheless, the U.S. hoped it might be an opportunity to keep the more liberal Medvedev, whom they considered more Western in his values, in power and have Putin relinquish the presidency for good. The U.S. appeared to make efforts to persuade Medvedev to stay on and called on Putin to let him stay in power. Vice President Joe Biden among others made this suggestion while on a trip to Moscow in 2011.

When Putin returned to the presidency in 2012, many observers believe he still felt Clinton's State Department had tried to meddle in Russian politics and specifically to threaten his presidency. From this perspective, allegedly interfering in Clinton’s own presidential race was a taste of her own medicine for Putin.

Of course, it is not publicly known how much of this speculation over bruised relations is really personal for Putin. More certain is there were reasons Putin might have wanted to delegitimize Clinton if she won or prevent her from winning if he could: As a candidate, Clinton had made it clear that she would take a hard line with Russia on sanctions over Ukraine and Syria and would continue to criticize Putin’s rule as illegitimate and authoritarian

Of course, publicly it’s not known how much of this speculation over bruised relations is really personal for Putin. But what is known is that Putin had every reason to want to delegitimize Clinton if she won or prevent her from winning if he could: She had made it very clear that she would take a hard line with Russia on sanctions and would continue to attack Putin’s presidency as illegitimate and authoritarian.

\ No newline at end of file diff --git a/src/extractors/custom/abcnews.go.com/index.js b/src/extractors/custom/abcnews.go.com/index.js new file mode 100644 index 000000000..7207378b9 --- /dev/null +++ b/src/extractors/custom/abcnews.go.com/index.js @@ -0,0 +1,51 @@ +export const AbcnewsGoComExtractor = { + domain: 'abcnews.go.com', + + title: { + selectors: [ + '.article-header h1', + ], + }, + + author: { + selectors: [ + '.authors', + ], + clean: [ + '.author-overlay', + '.by-text', + ], + }, + + date_published: { + selectors: [ + '.timestamp', + ], + timezone: 'America/New_York', + + }, + + lead_image_url: { + selectors: [ + ['meta[name="og:image"]', 'value'], + ], + }, + + content: { + selectors: [ + '.article-copy', + ], + + // Is there anything in the content you selected that needs transformed + // before it's consumable content? E.g., unusual lazy loaded images + transforms: { + }, + + // Is there anything that is in the result that shouldn't be? + // The clean selectors will remove anything that matches from + // the result + clean: [ + + ], + }, +}; diff --git a/src/extractors/custom/abcnews.go.com/index.test.js b/src/extractors/custom/abcnews.go.com/index.test.js new file mode 100644 index 000000000..7943e7c44 --- /dev/null +++ b/src/extractors/custom/abcnews.go.com/index.test.js @@ -0,0 +1,87 @@ +import assert from 'assert'; +import fs from 'fs'; +import URL from 'url'; +import cheerio from 'cheerio'; + +import Mercury from 'mercury'; +import getExtractor from 'extractors/get-extractor'; +import { excerptContent } from 'utils/text'; + +describe('AbcnewsGoComExtractor', () => { + describe('initial test case', () => { + let result; + let url; + beforeAll(() => { + url = + 'http://abcnews.go.com/US/hillary-clinton-putins-alleged-involvement-democratic-hack-stems/story?id=44233864&cid=clicksource_4380645_2_three_posts_vert_hed'; + const html = + fs.readFileSync('./fixtures/abcnews.go.com/1481922563840.html'); + result = + Mercury.parse(url, html, { fallback: false }); + }); + + it('is selected properly', () => { + // This test should be passing by default. + // It sanity checks that the correct parser + // is being selected for URLs from this domain + const extractor = getExtractor(url); + assert.equal(extractor.domain, URL.parse(url).hostname); + }); + + it('returns the title', async () => { + // To pass this test, fill out the title selector + // in ./src/extractors/custom/abcnews.go.com/index.js. + const { title } = await result; + + // Update these values with the expected values from + // the article. + assert.equal(title, 'Hillary Clinton: Putin\'s Alleged Involvement in Democratic Hack Stems From Longtime Grudge'); + }); + + it('returns the author', async () => { + // To pass this test, fill out the author selector + // in ./src/extractors/custom/abcnews.go.com/index.js. + const { author } = await result; + + // Update these values with the expected values from + // the article. + assert.equal(author, 'Josh Haskell David Caplan PATRICK REEVELL'); + }); + + it('returns the date_published', async () => { + // To pass this test, fill out the date_published selector + // in ./src/extractors/custom/abcnews.go.com/index.js. + const { date_published } = await result; + + // Update these values with the expected values from + // the article. + assert.equal(date_published, '2016-12-16T17:37:00.000Z'); + }); + + it('returns the lead_image_url', async () => { + // To pass this test, fill out the lead_image_url selector + // in ./src/extractors/custom/abcnews.go.com/index.js. + const { lead_image_url } = await result; + + // Update these values with the expected values from + // the article. + assert.equal(lead_image_url, 'http://a.abcnews.com/images/Politics/AP-hillary-clinton-01-as-161216_16x9_992.jpg'); + }); + + it('returns the content', async () => { + // To pass this test, fill out the content selector + // in ./src/extractors/custom/abcnews.go.com/index.js. + // You may also want to make use of the clean and transform + // options. + const { content } = await result; + + const $ = cheerio.load(content || ''); + + const first13 = excerptContent($('*').first().text(), 13); + + // Update these values with the expected values from + // the article. + assert.equal(first13, 'Hillary Clinton has an explanation for Vladimir Putin\'s alleged involvement in the hacking'); + }); + }); +}); diff --git a/src/extractors/custom/index.js b/src/extractors/custom/index.js index 221359569..b3481d9e3 100644 --- a/src/extractors/custom/index.js +++ b/src/extractors/custom/index.js @@ -37,6 +37,7 @@ export * from './www.reuters.com'; export * from './mashable.com'; export * from './www.chicagotribune.com'; export * from './www.vox.com'; +export * from './abcnews.go.com'; export * from './www.nydailynews.com'; export * from './www.cnbc.com'; export * from './www.popsugar.com';