diff --git a/ansible-docker-iframely/templates/config.local.js.j2 b/ansible-docker-iframely/templates/config.local.js.j2 index bde5aaa8c..bbf786599 100644 --- a/ansible-docker-iframely/templates/config.local.js.j2 +++ b/ansible-docker-iframely/templates/config.local.js.j2 @@ -1,5 +1,4 @@ -(function() { - var config = { + export default { // Specify a path for custom plugins. Custom plugins will override core plugins. // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder', @@ -259,7 +258,4 @@ } } */ - }; - - module.exports = config; -})(); + }; \ No newline at end of file diff --git a/app.js b/app.js index 08a9bf4fc..60f407f1e 100644 --- a/app.js +++ b/app.js @@ -1,4 +1,6 @@ -var sysUtils = require('./utils'); +import { cacheMiddleware, NotFound } from './utils.js'; +import CONFIG from './config.loader.js'; +global.CONFIG = CONFIG; console.log(""); console.log("Starting Iframely..."); @@ -8,13 +10,12 @@ if (!CONFIG.baseAppUrl) { console.warn('Warning: CONFIG.baseAppUrl not set, default value used'); } -var path = require('path'); -var express = require('express'); -var jsonxml = require('jsontoxml'); +import express from 'express'; +import * as jsonxml from 'jsontoxml'; -var NotFound = sysUtils.NotFound; +const app = express(); -var app = express(); +export default app; app.set('view engine', 'ejs'); @@ -40,12 +41,17 @@ app.use(function(req, res, next) { next(); }); -app.use(sysUtils.cacheMiddleware); +app.use(cacheMiddleware); +import apiViews from './modules/api/views.js'; +import debugViews from './modules/debug/views.js'; +apiViews(app); +debugViews(app); -require('./modules/api/views')(app); -require('./modules/debug/views')(app); -require('./modules/tests-ui/views')(app); +if (CONFIG.tests) { + const testViews = await import('./modules/tests-ui/views.js'); + testViews.default(app); +} app.use(logErrors); app.use(errorHandler); @@ -168,7 +174,4 @@ app.get('/', function(req, res) { res.end(); }); -process.title = "iframely"; -process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; - -module.exports = app; +process.title = "iframely"; \ No newline at end of file diff --git a/cluster.js b/cluster.js index 1f2ccc73d..1c7f393ea 100644 --- a/cluster.js +++ b/cluster.js @@ -1,5 +1,5 @@ -var GracefulCluster = require('graceful-cluster').GracefulCluster; -var sysUtils = require('./utils'); +import { GracefulCluster } from 'graceful-cluster'; +import * as sysUtils from './utils.js'; process.title = 'iframely-cluster'; @@ -10,6 +10,6 @@ GracefulCluster.start({ restartOnTimeout: CONFIG.CLUSTER_WORKER_RESTART_ON_PERIOD, restartOnMemory: CONFIG.CLUSTER_WORKER_RESTART_ON_MEMORY_USED, serverFunction: function() { - require('./server'); + import('./server'); } }); \ No newline at end of file diff --git a/config.js b/config.js index 32869fb93..6fd5c36e5 100644 --- a/config.js +++ b/config.js @@ -1,19 +1,18 @@ -(function() { + import * as _ from 'underscore'; + import * as path from 'path'; + import * as fs from 'fs'; - // Monkey patch before you require http for the first time. - var majorVersion = process.version.match(/v(\d+)\./); - majorVersion = parseInt(majorVersion); - if (majorVersion < 10) { - process.binding('http_parser').HTTPParser = require('http-parser-js').HTTPParser; - } - - var _ = require('underscore'); - var path = require('path'); - var fs = require('fs'); + import { fileURLToPath } from 'url'; + import { dirname } from 'path'; - var version = require('./package.json').version; + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); + + import { readFile } from 'fs/promises'; + const json = JSON.parse(await readFile(new URL('./package.json', import.meta.url))); + var version = json.version; - var config = { + const config = { baseAppUrl: "", port: 8061, @@ -383,7 +382,8 @@ // Providers config loader. var local_config_path = path.resolve(__dirname, "config.providers.js"); if (fs.existsSync(local_config_path)) { - var local = require(local_config_path); + var local = await import(local_config_path); + local = local && local.default; _.extend(config, local); } @@ -397,11 +397,13 @@ // Try config by NODE_ENV. if (fs.existsSync(env_config_path)) { - var local = require(env_config_path); + var local = await import(env_config_path); + local = local && local.default; } else if (fs.existsSync(local_config_path)) { // Else - try local config. - var local = require(local_config_path); + var local = await import(local_config_path); + local = local && local.default; } _.extend(config, local); @@ -429,5 +431,4 @@ config.HTTP2_RETRY_CODES[item] = 1; }); - module.exports = config; -})(); + export default config; \ No newline at end of file diff --git a/config.loader.js b/config.loader.js new file mode 100644 index 000000000..e7bf4c1a4 --- /dev/null +++ b/config.loader.js @@ -0,0 +1,5 @@ +import iframelyConfig from './config.js'; +// Load global config from exec dir, because `iframely` can be used as library. +var globalConfig = await import(process.cwd() + '/config.js'); +globalConfig = globalConfig && globalConfig.default; +export default {...iframelyConfig, ...globalConfig}; \ No newline at end of file diff --git a/config.local.js.SAMPLE b/config.local.js.SAMPLE index 02382972b..cf3c69de5 100644 --- a/config.local.js.SAMPLE +++ b/config.local.js.SAMPLE @@ -1,331 +1,332 @@ -(function() { - var config = { - - // Specify a path for custom plugins. Custom plugins will override core plugins. - // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder', - - DEBUG: false, - RICH_LOG_ENABLED: false, - - // For embeds that require render, baseAppUrl will be used as the host. - baseAppUrl: "http://localhost:8061", // use "https://yourdomain.com/path" where you have Iframely in your reverse proxy - relativeStaticUrl: "/r", - - // Or just skip built-in renders altogether - SKIP_IFRAMELY_RENDERS: true, - - // For legacy reasons the response format of Iframely open-source is - // different by default as it does not group the links array by rel. - // In order to get the same grouped response as in Cloud API, - // add `&group=true` to your request to change response per request - // or set `GROUP_LINKS` in your config to `true` for a global change. - GROUP_LINKS: true, - - // Number of maximum redirects to follow before aborting the page - // request with `redirect loop` error. - MAX_REDIRECTS: 4, - - SKIP_OEMBED_RE_LIST: [ - // /^https?:\/\/yourdomain\.com\//, +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; +import { readFile } from 'fs/promises'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// Optional SSL cert, if you serve under HTTPS. +/* +const ssl_key = const file = await readFile(new URL('./key.pem', import.meta.url), {encoding: 'utf8'}); +const ssl_cert = await readFile(new URL('./cert.pem', import.meta.url), {encoding: 'utf8'}); +*/ + +export default { + + // Specify a path for custom plugins. Custom plugins will override core plugins. + // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder', + + DEBUG: false, + RICH_LOG_ENABLED: false, + + // For embeds that require render, baseAppUrl will be used as the host. + baseAppUrl: "http://localhost:8061", // use "https://yourdomain.com/path" where you have Iframely in your reverse proxy + relativeStaticUrl: "/r", + + // Or just skip built-in renders altogether + SKIP_IFRAMELY_RENDERS: true, + + // For legacy reasons the response format of Iframely open-source is + // different by default as it does not group the links array by rel. + // In order to get the same grouped response as in Cloud API, + // add `&group=true` to your request to change response per request + // or set `GROUP_LINKS` in your config to `true` for a global change. + GROUP_LINKS: true, + + // Number of maximum redirects to follow before aborting the page + // request with `redirect loop` error. + MAX_REDIRECTS: 4, + + SKIP_OEMBED_RE_LIST: [ + // /^https?:\/\/yourdomain\.com\//, + ], + + /* + // Used to pass parameters to the generate functions when creating HTML elements + // disableSizeWrapper: Don't wrap element (iframe, video, etc) in a positioned div + GENERATE_LINK_PARAMS: { + disableSizeWrapper: true + }, + */ + + port: 8061, //can be overridden by PORT env var + host: '0.0.0.0', // Dockers beware. See https://github.com/itteco/iframely/issues/132#issuecomment-242991246 + //can be overridden by HOST env var + + // Optional SSL cert, if you serve under HTTPS. + /* + ssl: { + key: ssl_key, + cert: ssl_cert, + port: 443 + }, + */ + + /* + Supported cache engines: + - no-cache - no caching will be used. + - node-cache - good for debug, node memory will be used (https://github.com/tcs-de/nodecache). + - redis - https://github.com/mranney/node_redis. + - memcached - https://github.com/3rd-Eden/node-memcached + */ + CACHE_ENGINE: 'node-cache', + CACHE_TTL: 0, // In seconds. + // 0 = 'never expire' for memcached & node-cache to let cache engine decide itself when to evict the record + // 0 = 'no cache' for redis. Use high enough (e.g. 365*24*60*60*1000) ttl for similar 'never expire' approach instead + + /* + // Redis mode (cluster or standard) + REDIS_MODE: 'standard', + */ + + /* + // Redis cache options. + REDIS_OPTIONS: { + host: '127.0.0.1', + port: 6379 + }, + */ + + /* + // Redis cluster options. + REDIS_CLUSTER_OPTIONS: { + servers: [ + { + host: '10.0.0.10', + port: 6379 + }, + // ... + ], + }, + */ + + /* + // Memcached options. See https://github.com/3rd-Eden/node-memcached#server-locations + MEMCACHED_OPTIONS: { + locations: "127.0.0.1:11211" + } + */ + + /* + // Access-Control-Allow-Origin list. + allowedOrigins: [ + "*", + "http://another_domain.com" + ], + */ + + /* + // Uncomment to enable plugin testing framework. + tests: { + mongodb: 'mongodb://localhost:27017/iframely-tests', + single_test_timeout: 10 * 1000, + plugin_test_period: 2 * 60 * 60 * 1000, + relaunch_script_period: 5 * 60 * 1000 + }, + */ + + // If there's no response from remote server, the timeout will occur after + RESPONSE_TIMEOUT: 5 * 1000, //ms + + // Customize API calls to oembed endpoints. + // Must have: please add your `access_token` for Facebook and Instagram API calls + ADD_OEMBED_PARAMS: [{ + + re: [ // Endpoint's URL regexp array. + /^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/instagram_oembed/i ], + params: { // Custom query-string params object. - /* - // Used to pass parameters to the generate functions when creating HTML elements - // disableSizeWrapper: Don't wrap element (iframe, video, etc) in a positioned div - GENERATE_LINK_PARAMS: { - disableSizeWrapper: true - }, - */ - - port: 8061, //can be overridden by PORT env var - host: '0.0.0.0', // Dockers beware. See https://github.com/itteco/iframely/issues/132#issuecomment-242991246 - //can be overridden by HOST env var + // TODO: get your access Insagtam token as described + // on https://developers.facebook.com/docs/instagram/oembed/ + access_token: '', // The simplest way is + // to use `{app-id}|{app secret}` as access token - // Optional SSL cert, if you serve under HTTPS. - /* - ssl: { - key: require('fs').readFileSync(__dirname + '/key.pem'), - cert: require('fs').readFileSync(__dirname + '/cert.pem'), - port: 443 + // Add any other optional params + hidecaption: true + } + }, { + re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_page/i], + params: { + // TODO: get your access token as described + // on https://developers.facebook.com/docs/plugins/oembed + access_token: '', // The simplest way is + // to use `{app-id}|{app secret}` as access token + + // Add any other optional params + show_posts: 0, + show_facepile: 0, + maxwidth: 600 + } + }, { + // match i=user or i=moment or i=timeline to configure these types invidually + // see params spec at https://dev.twitter.com/web/embedded-timelines/oembed + re: [/^https?:\/\/publish\.twitter\.com\/oembed\?i=user/i], + params: { + limit: 1, + maxwidth: 600 + } + }, { + // Facebook https://developers.facebook.com/docs/plugins/oembed/ + re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_/i], + params: { + // TODO: get your access token as described + // on https://developers.facebook.com/docs/plugins/oembed + access_token: '', // The simplest way is + // to use `{app-id}|{app secret}` as access token + + // Add any other optional params, like skip script tag and fb-root div + // omitscript: true + } + }], + + /* Configure use of HTTP proxies as needed. + You don't have to specify all options per regex - just what you need to override + */ + /* + PROXY: [{ + re: [/^https?:\/\/www\.domain\.com/], + proxy_server: 'http://1.2.3.4:8080', + user_agent: 'CHANGE YOUR AGENT', + headers: { + // HTTP headers + // Overrides previous params if overlapped. }, - */ - - /* - Supported cache engines: - - no-cache - no caching will be used. - - node-cache - good for debug, node memory will be used (https://github.com/tcs-de/nodecache). - - redis - https://github.com/mranney/node_redis. - - memcached - https://github.com/3rd-Eden/node-memcached - */ - CACHE_ENGINE: 'node-cache', - CACHE_TTL: 0, // In seconds. - // 0 = 'never expire' for memcached & node-cache to let cache engine decide itself when to evict the record - // 0 = 'no cache' for redis. Use high enough (e.g. 365*24*60*60*1000) ttl for similar 'never expire' approach instead - - /* - // Redis mode (cluster or standard) - REDIS_MODE: 'standard', - */ - - /* - // Redis cache options. - REDIS_OPTIONS: { - host: '127.0.0.1', - port: 6379 + request_options: { + // Refer to: https://github.com/request/request + // Overrides previous params if overlapped. }, - */ - - /* - // Redis cluster options. - REDIS_CLUSTER_OPTIONS: { - servers: [ - { - host: '10.0.0.10', - port: 6379 - }, - // ... - ], + cache_ttl: 3600 // in seconds, cache response for 1 hour. + }], + */ + + // Customize API calls to 3rd parties. At the very least - configure required keys. + // For available provider options - please see the code of its domain plugin. + providerOptions: { + locale: "en_US", // ISO 639-1 two-letter language code, e.g. en_CA or fr_CH. + // Will be added as highest priotity in accept-language header with each request. + // Plus is used in FB, YouTube and perhaps other plugins + "twitter": { + "max-width": 550, + "min-width": 250, + hide_media: false, + hide_thread: false, + omit_script: false, + center: false, + // dnt: true, + cache_ttl: 100 * 365 * 24 * 3600 // 100 Years. }, - */ - - /* - // Memcached options. See https://github.com/3rd-Eden/node-memcached#server-locations - MEMCACHED_OPTIONS: { - locations: "127.0.0.1:11211" - } - */ - - /* - // Access-Control-Allow-Origin list. - allowedOrigins: [ - "*", - "http://another_domain.com" - ], - */ - - /* - // Uncomment to enable plugin testing framework. - tests: { - mongodb: 'mongodb://localhost:27017/iframely-tests', - single_test_timeout: 10 * 1000, - plugin_test_period: 2 * 60 * 60 * 1000, - relaunch_script_period: 5 * 60 * 1000 + readability: { + enabled: false + // allowPTagDescription: true // to enable description fallback to first paragraph }, - */ - - // If there's no response from remote server, the timeout will occur after - RESPONSE_TIMEOUT: 5 * 1000, //ms - - /* From v1.4.0, Iframely works over HTTP/2 if not disabled. - Alternatively, you can also disable per origin. See `proxy` option below. - The default in this sample config is to disable HTTP/2 as per - https://github.com/itteco/iframely/issues/277. - */ - DISABLE_HTTP2: true, - - // Customize API calls to oembed endpoints. - // Must have: please add your `access_token` for Facebook and Instagram API calls - ADD_OEMBED_PARAMS: [{ - - re: [ // Endpoint's URL regexp array. - /^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/instagram_oembed/i - ], - params: { // Custom query-string params object. - - // TODO: get your access Insagtam token as described - // on https://developers.facebook.com/docs/instagram/oembed/ - access_token: '', // The simplest way is - // to use `{app-id}|{app secret}` as access token - - // Add any other optional params - hidecaption: true - } - }, { - re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_page/i], - params: { - // TODO: get your access token as described - // on https://developers.facebook.com/docs/plugins/oembed - access_token: '', // The simplest way is - // to use `{app-id}|{app secret}` as access token - - // Add any other optional params - show_posts: 0, - show_facepile: 0, - maxwidth: 600 - } - }, { - // match i=user or i=moment or i=timeline to configure these types invidually - // see params spec at https://dev.twitter.com/web/embedded-timelines/oembed - re: [/^https?:\/\/publish\.twitter\.com\/oembed\?i=user/i], - params: { - limit: 1, - maxwidth: 600 - } - }, { - // Facebook https://developers.facebook.com/docs/plugins/oembed/ - re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_/i], - params: { - // TODO: get your access token as described - // on https://developers.facebook.com/docs/plugins/oembed - access_token: '', // The simplest way is - // to use `{app-id}|{app secret}` as access token - - // Add any other optional params, like skip script tag and fb-root div - // omitscript: true - } - }], + images: { + loadSize: false, // if true, will try an load first bytes of all images to get/confirm the sizes + checkFavicon: false // if true, will verify all favicons + }, + tumblr: { + consumer_key: "INSERT YOUR VALUE" + // media_only: true // disables status embeds for images and videos - will return plain media + }, + google: { + // https://developers.google.com/maps/documentation/embed/guide#api_key + maps_key: "INSERT YOUR VALUE" + }, - /* Configure use of HTTP proxies as needed. - You don't have to specify all options per regex - just what you need to override - */ /* - PROXY: [{ - re: [/^https?:\/\/www\.domain\.com/], - proxy_server: 'http://1.2.3.4:8080', - user_agent: 'CHANGE YOUR AGENT', - headers: { - // HTTP headers - // Overrides previous params if overlapped. - }, - request_options: { - // Refer to: https://github.com/request/request - // Overrides previous params if overlapped. - }, - cache_ttl: 3600, // in seconds, cache response for 1 hour. - disable_http2: true - }], + // Optional Camo Proxy to wrap all images: https://github.com/atmos/camo + camoProxy: { + camo_proxy_key: "INSERT YOUR VALUE", + camo_proxy_host: "INSERT YOUR VALUE" + // ssl_only: true // will only proxy non-ssl images + }, */ - // Customize API calls to 3rd parties. At the very least - configure required keys. - // For available provider options - please see the code of its domain plugin. - providerOptions: { - locale: "en_US", // ISO 639-1 two-letter language code, e.g. en_CA or fr_CH. - // Will be added as highest priotity in accept-language header with each request. - // Plus is used in FB, YouTube and perhaps other plugins - "twitter": { - "max-width": 550, - "min-width": 250, - hide_media: false, - hide_thread: false, - omit_script: false, - center: false, - // dnt: true, - cache_ttl: 100 * 365 * 24 * 3600 // 100 Years. - }, - readability: { - enabled: false - // allowPTagDescription: true // to enable description fallback to first paragraph - }, - images: { - loadSize: false, // if true, will try an load first bytes of all images to get/confirm the sizes - checkFavicon: false // if true, will verify all favicons - }, - tumblr: { - consumer_key: "INSERT YOUR VALUE" - // media_only: true // disables status embeds for images and videos - will return plain media - }, - google: { - // https://developers.google.com/maps/documentation/embed/guide#api_key - maps_key: "INSERT YOUR VALUE" - }, - - /* - // Optional Camo Proxy to wrap all images: https://github.com/atmos/camo - camoProxy: { - camo_proxy_key: "INSERT YOUR VALUE", - camo_proxy_host: "INSERT YOUR VALUE" - // ssl_only: true // will only proxy non-ssl images - }, - */ - - // List of query parameters to add to YouTube and Vimeo frames - // Start it with leading "?". Or omit alltogether for default values - // API key is optional, youtube will work without it too. - // It is probably the same API key you use for Google Maps. - youtube: { - // api_key: "INSERT YOUR VALUE", - // parts: [ "snippet", "player" ], // list of fields you want to use in the request, in most cases you only need those two - get_params: "?rel=0&showinfo=1" // https://developers.google.com/youtube/player_parameters - }, - vimeo: { - get_params: "?byline=0&badge=0" // https://developer.vimeo.com/player/embedding - }, - soundcloud: { - old_player: true // enables classic player - }, - giphy: { - media_only: true // disables branded player for gifs and returns just the image - }, - bandcamp: { - get_params: '/size=large/bgcol=333333/linkcol=ffffff/artwork=small/transparent=true/', - media: { - album: { - height: 472, - 'max-width': 700 - }, - track: { - height: 120, - 'max-width': 700 - } + // List of query parameters to add to YouTube and Vimeo frames + // Start it with leading "?". Or omit alltogether for default values + // API key is optional, youtube will work without it too. + // It is probably the same API key you use for Google Maps. + youtube: { + // api_key: "INSERT YOUR VALUE", + // parts: [ "snippet", "player" ], // list of fields you want to use in the request, in most cases you only need those two + get_params: "?rel=0&showinfo=1" // https://developers.google.com/youtube/player_parameters + }, + vimeo: { + get_params: "?byline=0&badge=0" // https://developer.vimeo.com/player/embedding + }, + soundcloud: { + old_player: true // enables classic player + }, + giphy: { + media_only: true // disables branded player for gifs and returns just the image + }, + bandcamp: { + get_params: '/size=large/bgcol=333333/linkcol=ffffff/artwork=small/transparent=true/', + media: { + album: { + height: 472, + 'max-width': 700 + }, + track: { + height: 120, + 'max-width': 700 } - }, - // Docs: https://dev.twitch.tv/docs/embed/video-and-clips - /* - twitch: { - parent: 'jsbin.com, null.jsbin.com, localhost' - }, - */ + } }, - - // WHITELIST_WILDCARD, if present, will be added to whitelist as record for top level domain: "*" - // with it, you can define what parsers do when they run accross unknown publisher. - // If absent or empty, all generic media parsers will be disabled except for known domains - // More about format: https://iframely.com/docs/qa-format - + // Docs: https://dev.twitch.tv/docs/embed/video-and-clips /* - WHITELIST_WILDCARD: { - "twitter": { - "player": "allow", - "photo": "deny" - }, - "oembed": { - "video": "allow", - "photo": "allow", - "rich": "deny", - "link": "deny" - }, - "og": { - "video": ["allow", "ssl", "responsive"] - }, - "iframely": { - "survey": "allow", - "reader": "allow", - "player": "allow", - "image": "allow" - }, - "html-meta": { - "video": ["allow", "responsive"], - "promo": "allow" - } - } + twitch: { + parent: 'jsbin.com, null.jsbin.com, localhost' + }, */ - - // The list of regexs to be ignored. Iframely will return 417 - // At minimum, keep your localhosts blacklisted to avoid SSRF - IGNORE_DOMAINS_RE: [ - /^https?:\/\/127\.0\.0\.1/i, - /^https?:\/\/localhost/i, - /^https?:\/\/[^\/]+:\d+\/?/, // Blocks port-scan via DNS pointing to 127.0.0.1 - - // And this is AWS metadata service - // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html - /^https?:\/\/169\.254\.169\.254/ - ], - - // Endpoint for prerender service, if you need it. Used to parse React apps. Very slow. - // Tested with https://github.com/prerender/prerender - // PRERENDER_URL: "https://domain/render?url=" - }; - - module.exports = config; -})(); + }, + + // WHITELIST_WILDCARD, if present, will be added to whitelist as record for top level domain: "*" + // with it, you can define what parsers do when they run accross unknown publisher. + // If absent or empty, all generic media parsers will be disabled except for known domains + // More about format: https://iframely.com/docs/qa-format + + /* + WHITELIST_WILDCARD: { + "twitter": { + "player": "allow", + "photo": "deny" + }, + "oembed": { + "video": "allow", + "photo": "allow", + "rich": "deny", + "link": "deny" + }, + "og": { + "video": ["allow", "ssl", "responsive"] + }, + "iframely": { + "survey": "allow", + "reader": "allow", + "player": "allow", + "image": "allow" + }, + "html-meta": { + "video": ["allow", "responsive"], + "promo": "allow" + } + } + */ + + // The list of regexs to be ignored. Iframely will return 417 + // At minimum, keep your localhosts blacklisted to avoid SSRF + IGNORE_DOMAINS_RE: [ + /^https?:\/\/127\.0\.0\.1/i, + /^https?:\/\/localhost/i, + /^https?:\/\/[^\/]+:\d+\/?/, // Blocks port-scan via DNS pointing to 127.0.0.1 + + // And this is AWS metadata service + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html + /^https?:\/\/169\.254\.169\.254/ + ], + + // Endpoint for prerender service, if you need it. Used to parse React apps. Very slow. + // Tested with https://github.com/prerender/prerender + // PRERENDER_URL: "https://domain/render?url=" +}; \ No newline at end of file diff --git a/config.test.js b/config.test.js index 1a4ee3ee1..0461a4ef4 100644 --- a/config.test.js +++ b/config.test.js @@ -1,7 +1,4 @@ -(function() { - - var config = { - +export default { DEBUG: true, RICH_LOG_ENABLED: true, @@ -11,7 +8,4 @@ RESPONSE_TIMEOUT: 1 * 100, //ms IGNORE_DOMAINS_RE: /blacklisted.*/ - }; - - module.exports = config; -})(); \ No newline at end of file +}; \ No newline at end of file diff --git a/config.test_with_custom_plugins.js b/config.test_with_custom_plugins.js index 2e36a421a..690852dbe 100644 --- a/config.test_with_custom_plugins.js +++ b/config.test_with_custom_plugins.js @@ -1,6 +1,4 @@ -(function() { - - var config = { + export default { CUSTOM_PLUGINS_PATH: __dirname + '/test/fixtures/custom-plugins', @@ -14,7 +12,4 @@ RESPONSE_TIMEOUT: 1 * 100, //ms IGNORE_DOMAINS_RE: /blacklisted.*/ - }; - - module.exports = config; -})(); \ No newline at end of file + }; \ No newline at end of file diff --git a/docs/WRITE-A-PLUGIN.md b/docs/WRITE-A-PLUGIN.md index e00c903e6..55e4b1c7e 100644 --- a/docs/WRITE-A-PLUGIN.md +++ b/docs/WRITE-A-PLUGIN.md @@ -125,7 +125,7 @@ See available attributes names to check if similar name exists at [/meta-mapping See example [/generic/meta/video.js](https://github.com/itteco/iframely/blob/master/plugins/generic/meta/video.js): - module.exports = { + export default { getMeta: function(meta) { // This prevents non useful errors loging with "undefined". @@ -150,13 +150,13 @@ But `og:title` usually better and contains only article title without site name. If you want to mark you plugin as worst source of meta (like html `` tag), use `lowestPriority: true`: - module.exports = { + export default { lowestPriority: true } If you want to mark your plugin as good source of meta (like og:title), use `highestPriority: true`: - module.exports = { + export default { highestPriority: true } diff --git a/lib/agent-http2.js b/lib/agent-http2.js index 027b0856c..ccc2df57b 100644 --- a/lib/agent-http2.js +++ b/lib/agent-http2.js @@ -1,25 +1,12 @@ -var semver = require('semver'); -var https = require('https'); -var http2; +import * as semver from 'semver'; +import * as https from 'https'; -if (semver.gte(process.version, '8.13.0')) { - - http2 = require('http2'); - -} else { - - // Old node without stable http2 support: fallback to https agent. - exports.Http2Agent = https.Agent; - return; -} - -var stream = require('stream'); -var util = require('util'); -var http = require('http'); -var tls = require('tls'); -var EventEmitter = require('events').EventEmitter; - -var sysUtils = require('../logging'); +import * as http2 from 'http2'; +import * as stream from 'stream'; +import * as util from 'util'; +import * as http from 'http'; +import * as tls from 'tls'; +import { EventEmitter } from 'events'; // Http2Response Class. @@ -38,7 +25,9 @@ function Http2Response(clientHttp2Stream) { clientHttp2Stream.on('response', (headers) => { + // TODO: status? that.statusCode = headers[':status']; + // TODO: headers wrapper? that.headers = headers; that.emit('ready'); @@ -89,7 +78,7 @@ Http2Response.prototype._transform = function (chunk, enc, cb) { // Http2Agent Class -function Http2Agent(options) { +export function Http2Agent(options) { if (!(this instanceof Http2Agent)) return new Http2Agent(options); @@ -378,10 +367,6 @@ Http2Agent.prototype.createConnection = function(options, tlsSocketCb, http2Sess }); }; -exports.Http2Agent = Http2Agent; - - - // HTTPS agents. Copy from https.js. const { inherits } = util; @@ -423,4 +408,3 @@ function CustomHttpsAgent(options) { } inherits(CustomHttpsAgent, https.Agent); CustomHttpsAgent.prototype.createConnection = createConnection; - \ No newline at end of file diff --git a/lib/cache-engines/memcached.js b/lib/cache-engines/memcached.js index d0cce3c47..7105c9f79 100644 --- a/lib/cache-engines/memcached.js +++ b/lib/cache-engines/memcached.js @@ -1,138 +1,133 @@ -(function(engine) { +import log from '../../logging.js'; +import * as crypto from 'crypto'; +import Memcached from 'memcached'; +import CONFIG from '../../config.loader.js'; - var sysUtils = require('../../logging'); +var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options); - var crypto = require('crypto'); +var timeout = CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options && CONFIG.MEMCACHED_OPTIONS.options.timeout; - var Memcached = require('memcached'); - var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options); +var MEMCACHED_MAX_DATA_SIZE = 1024 * 1024; // Megabyte. - var timeout = CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options && CONFIG.MEMCACHED_OPTIONS.options.timeout; +function safeKey(key) { + return crypto.createHash('md5').update(key).digest("hex"); +} - var MEMCACHED_MAX_DATA_SIZE = 1024 * 1024; // Megabyte. +function _findKeyMeta(k) { - function safeKey(key) { - return crypto.createHash('md5').update(key).digest("hex"); - } - - function _findKeyMeta(k) { - var _ = require('underscore'); + var sk = safeKey(k); - var sk = safeKey(k); + memcached.items(function(a, b){ - memcached.items(function(a, b){ + var stubs = Object.keys(b[0]); - var stubs = _.keys(b[0]); + stubs.forEach(function(sid) { - stubs.forEach(function(sid) { + var servers = CONFIG.MEMCACHED_OPTIONS.locations; + if (servers instanceof Object) { + servers = Object.keys(servers); + } + if (typeof servers === 'string') { + servers = [servers]; + } - var servers = CONFIG.MEMCACHED_OPTIONS.locations; - if (servers instanceof Object) { - servers = _.keys(servers); - } - if (typeof servers === 'string') { - servers = [servers]; - } + servers.forEach(function(s) { + memcached.cachedump(s, parseInt(sid), 0, function(a, b) { - servers.forEach(function(s) { - memcached.cachedump(s, parseInt(sid), 0, function(a, b) { + if (!(b instanceof Array)) { + b = [b]; + } - if (!(b instanceof Array)) { - b = [b]; + b.forEach(function(k) { + if (k.key === sk) { + console.log(' - key', k); + console.log(' - now is', new Date()); + console.log(' - exp in', new Date(k.s * 1000)); } - - b.forEach(function(k) { - if (k.key === sk) { - console.log(' - key', k); - console.log(' - now is', new Date()); - console.log(' - exp in', new Date(k.s * 1000)); - } - }) - }); + }) }); }); }); - } - - engine.set = function(_key, data, options) { + }); +} - var key = (!options || !options.raw) ? safeKey(_key) : _key; +export function set(_key, data, options) { - //console.log(key, (!options || !options.raw) ? JSON.stringify(data) : data); + var key = (!options || !options.raw) ? safeKey(_key) : _key; - // Warning: value and lifetime argumets switched, bug in docs. - // Warning: need replace /n if raw saved. Memcached in nginx read bug. - var storedData = (!options || !options.raw) ? JSON.stringify(data) : data.replace(/\n/g, ''); + //console.log(key, (!options || !options.raw) ? JSON.stringify(data) : data); - if (storedData && storedData.length > MEMCACHED_MAX_DATA_SIZE) { + // Warning: value and lifetime argumets switched, bug in docs. + // Warning: need replace /n if raw saved. Memcached in nginx read bug. + var storedData = (!options || !options.raw) ? JSON.stringify(data) : data.replace(/\n/g, ''); - sysUtils.log(' -- Memcached handled set error ' + _key + ': The length of the value is ' + storedData.length + ' and greater than ' + MEMCACHED_MAX_DATA_SIZE); + if (storedData && storedData.length > MEMCACHED_MAX_DATA_SIZE) { - } else { + log(' -- Memcached handled set error ' + _key + ': The length of the value is ' + storedData.length + ' and greater than ' + MEMCACHED_MAX_DATA_SIZE); - memcached.set(key, storedData, options && options.ttl || CONFIG.CACHE_TTL, function(error){ - if (error) { - sysUtils.log(' -- Memcached set error ' + _key + ' ' + error); - } - }); - } - }; + } else { - engine.get = function(_key, cb) { - - var key = safeKey(_key); + memcached.set(key, storedData, options && options.ttl || CONFIG.CACHE_TTL, function(error){ + if (error) { + log(' -- Memcached set error ' + _key + ' ' + error); + } + }); + } +}; - var timeoutId, finished = false; +export function get(_key, cb) { - if (timeout) { - setTimeout(function() { - if (finished) { - return; - } - finished = true; - sysUtils.log(' -- Memcached soft timeout on get ' + _key); - cb(null, null); - }, timeout); - } + var key = safeKey(_key); - memcached.get(key, function (error, data) { + var timeoutId, finished = false; + if (timeout) { + setTimeout(function() { if (finished) { return; } finished = true; + log(' -- Memcached soft timeout on get ' + _key); + cb(null, null); + }, timeout); + } - clearTimeout(timeoutId); + memcached.get(key, function (error, data) { - if (error) { - sysUtils.log(' -- Memcached get error ' + _key + ' ' + error); - // Fail silent. - return cb(null, null); - } + if (finished) { + return; + } + finished = true; - if (typeof data !== 'string') { - return cb(null, data); - } + clearTimeout(timeoutId); - try { - var parsedData = JSON.parse(data); - } catch(ex) { - sysUtils.log(' -- Memcached: error JSON parse value ' + _key + ' ' + ex.message); - return cb(null, null); - } + if (error) { + log(' -- Memcached get error ' + _key + ' ' + error); + // Fail silent. + return cb(null, null); + } - if (parsedData && typeof parsedData === 'object' && _key in parsedData) { - // Support old multi cached data. - parsedData = parsedData[_key]; - } + if (typeof data !== 'string') { + return cb(null, data); + } - cb(null, parsedData); - }); - }; + try { + var parsedData = JSON.parse(data); + } catch(ex) { + log(' -- Memcached: error JSON parse value ' + _key + ' ' + ex.message); + return cb(null, null); + } + + if (parsedData && typeof parsedData === 'object' && _key in parsedData) { + // Support old multi cached data. + parsedData = parsedData[_key]; + } - engine.getClient = function() { - return memcached; - }; + cb(null, parsedData); + }); +}; -})(exports); \ No newline at end of file +export function getClient() { + return memcached; +}; diff --git a/lib/cache-engines/no-cache.js b/lib/cache-engines/no-cache.js index 2bd2cc32b..500c6ebb3 100644 --- a/lib/cache-engines/no-cache.js +++ b/lib/cache-engines/no-cache.js @@ -1,11 +1,7 @@ -(function(engine) { - - engine.set = function(key, data) { + export function set(key, data) { }; - engine.get = function(key, cb) { + export function get(key, cb) { cb(null, null); - }; - -})(exports); \ No newline at end of file + }; \ No newline at end of file diff --git a/lib/cache-engines/node-cache.js b/lib/cache-engines/node-cache.js index 48227c3ee..823b21a15 100644 --- a/lib/cache-engines/node-cache.js +++ b/lib/cache-engines/node-cache.js @@ -1,16 +1,16 @@ -(function(cache) { + import NodeCache from "node-cache"; + import CONFIG from '../../config.loader.js'; - var NodeCache = require("node-cache"); var nodeCache = new NodeCache({ stdTTL: CONFIG.CACHE_TTL, checkperiod: CONFIG.CACHE_TTL / 2 }); - cache.set = function(key, data, options) { + export function set(key, data, options) { nodeCache.set(key, data, options && options.ttl || CONFIG.CACHE_TTL); - }; + } - cache.get = function(key, cb) { + export function get(key, cb) { nodeCache.get(key, function(error, data) { if (error) { return cb(null, null); @@ -22,6 +22,4 @@ cb(null, null); } }); - }; - -})(exports); \ No newline at end of file + } \ No newline at end of file diff --git a/lib/cache-engines/redis.js b/lib/cache-engines/redis.js index 451515885..8b03b1dd6 100644 --- a/lib/cache-engines/redis.js +++ b/lib/cache-engines/redis.js @@ -1,36 +1,35 @@ -(function(engine) { - - var sysUtils = require('../../logging'); + import log from '../../logging.js'; + import CONFIG from '../../config.loader.js'; var client; if (CONFIG.REDIS_MODE === 'cluster') { - var RedisClustr = require('redis-clustr'); - + const pkg = await import('redis-clustr'); + const RedisClustr = pkg.default; client = new RedisClustr(CONFIG.REDIS_CLUSTER_OPTIONS); } else { - var redis = require('redis'); - client = redis.createClient(CONFIG.REDIS_OPTIONS); + var pkg = await import('redis'); + client = pkg.createClient(CONFIG.REDIS_OPTIONS); } - engine.set = function(key, data, options) { + export function set(key, data, options) { var multi = client.multi(); multi.set(key, JSON.stringify(data)); multi.expire(key, options && options.ttl || CONFIG.CACHE_TTL); multi.exec(function(error) { if (error) { - sysUtils.log(' -- Redis set error ' + key + ' ' + error); + log(' -- Redis set error ' + key + ' ' + error); } }); }; - engine.get = function(key, cb) { + export function get(key, cb) { client.get(key, function(error, data) { if (error) { - sysUtils.log(' -- Redis get error ' + key + ' ' + error); + log(' -- Redis get error ' + key + ' ' + error); return cb(null, null); } @@ -46,6 +45,4 @@ cb(null, parsedData); }); - }; - -})(exports); + }; \ No newline at end of file diff --git a/lib/cache.js b/lib/cache.js index 5169cb7fe..470e4a49d 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -1,24 +1,25 @@ -(function(cache) { - - var _ = require('underscore'); + import * as _ from 'underscore'; + import CONFIG from '../config.loader.js'; var DEFAULT_CACHE = "node-cache"; - function setCachingEngine(path) { + export const cache = {}; + + async function setCachingEngine(path) { try { if (path.indexOf('/') === -1) { // Not full path. - path = './cache-engines/' + path; + path = './cache-engines/' + path + '.js'; } var id = path.split('/').slice(-1)[0].replace(/\.js$/, ''); - var cache_engine = require(path); + var cache_engine = await import(path); if (!cache_engine.set || !cache_engine.get) { console.warn("Default cache engine used. No get and set methods in cache engine", id); - return setCachingEngine(DEFAULT_CACHE); + return await setCachingEngine(DEFAULT_CACHE); } cache.set = function(key, data, options) { @@ -38,16 +39,16 @@ console.log("Using cache engine:", id); } catch (ex) { - if (path == DEFAULT_CACHE) { + if (path.indexOf(DEFAULT_CACHE) > -1) { throw ex; } else { console.warn("Default cache engine used. Check CONFIG.CACHE_ENGINE. ", ex.stack); - setCachingEngine(DEFAULT_CACHE); + await setCachingEngine(DEFAULT_CACHE); } } } - setCachingEngine(CONFIG.CACHE_ENGINE || DEFAULT_CACHE); + await setCachingEngine(CONFIG.CACHE_ENGINE || DEFAULT_CACHE); cache.withCache = function(key, func, options, callback) { @@ -142,7 +143,7 @@ * * */ - var MultiCache = cache.MultiCache = function(options) { + const MultiCache = cache.MultiCache = function(options) { this.options = options; this.keysDict = {}; @@ -184,9 +185,9 @@ var that = this; - var getsCount = _.keys(this.getCbDict).length; + var getsCount = Object.keys(this.getCbDict).length; - var keysList = _.keys(this.keysDict).sort(); + var keysList = Object.keys(this.keysDict).sort(); this.multiKey = keysList.join(':'); if (keysList.length > 1) { this.multiKey = 'multi:' + this.multiKey; @@ -262,7 +263,7 @@ }; MultiCache.prototype._checkAllSet = function() { - if (_.keys(this.keysDict).length === _.keys(this.setKeysDict).length && this.multiKey) { + if (Object.keys(this.keysDict).length === Object.keys(this.setKeysDict).length && this.multiKey) { this._runAllSets(); } }; @@ -285,6 +286,4 @@ } cache.set(this.multiKey, data, this.options); - }; - -})(exports); \ No newline at end of file + }; \ No newline at end of file diff --git a/lib/core.js b/lib/core.js index 1db05765e..8e5e69afa 100644 --- a/lib/core.js +++ b/lib/core.js @@ -1,21 +1,19 @@ -(function(core) { - var _ = require('underscore'), - request = require('request'), - urlLib = require('url'); + import * as _ from 'underscore'; + import * as urlLib from 'url'; + import * as pluginUtils from './loader/utils.js'; + import * as utils from './utils.js'; + import log from '../logging.js'; + import * as oembedUtils from './oembed.js'; + import * as pluginLoader from './loader/pluginLoader.js'; + import requestWrapper from './request.js'; - var pluginUtils = require('./loader/utils'), - utils = require('./utils'), - sysUtils = require('../logging'), - oembedUtils = require('./oembed'), - pluginLoader = require('./loader/pluginLoader'), - requestWrapper = require('./request'); - - var plugins = pluginLoader._plugins, + const plugins = pluginLoader._plugins, providedParamsDict = pluginLoader._providedParamsDict, pluginsList = pluginLoader._pluginsList, usedParamsDict = pluginLoader._usedParamsDict, postPluginsList = pluginLoader._postPluginsList, + templates = pluginLoader._templates, PLUGIN_METHODS = pluginUtils.PLUGIN_METHODS; @@ -33,7 +31,7 @@ * * * loadedParams - list of currently loaded params keys. - * var loadedParams = _.keys(context); + * var loadedParams = Object.keys(context); * loadedParams = ['cb', 'uri', 'title', 'meta'] * * @@ -80,7 +78,7 @@ * mandatoryParams - list of new params not used by plugins. Core will find what can use them. * 'mandatoryParams' enables mandatory mode: function will use _only_ methods which has this input 'mandatoryParams'. * This is used for "go down by tree" algorithm. - * var mandatoryParams = _.difference(loadedParams, _.keys(usedParams)); + * var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams)); * mandatoryParams = [ * paramName * ] @@ -347,9 +345,9 @@ * */ function runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb) { - var loadedParams = _.keys(context); + var loadedParams = Object.keys(context); - var mandatoryParams = _.difference(loadedParams, _.keys(usedParams)); + // var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams)); // Reset scanned plugins for each iteration. var scannedPluginsIds = {}; @@ -364,16 +362,16 @@ } // If has new unused params (mandatoryParams) - then find plugins which can use them. - if (mandatoryParams && mandatoryParams.length > 0) { + // if (mandatoryParams && mandatoryParams.length > 0) { - var secondaryPlugins = findPluginsForMandatoryParams(mandatoryParams, usedDomains); + // var secondaryPlugins = findPluginsForMandatoryParams(mandatoryParams, usedDomains); - // Find methods in plugins, which can use mandatory params. - for(var i = 0; i < secondaryPlugins.length; i++) { - var pluginId = secondaryPlugins[i]; - findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams); - } - } + // // Find methods in plugins, which can use mandatory params. + // for(var i = 0; i < secondaryPlugins.length; i++) { + // var pluginId = secondaryPlugins[i]; + // findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams); + // } + // } // Run found methods. runMethods(methods, context, pluginsUrlMatches, options, asyncMethodCb); @@ -750,11 +748,18 @@ continue; - } else if (param === 'pluginId') { + } else if (param === 'plugin') { + + args.push(plugins[dataRecord.method.pluginId]); + + continue; + + } else if (param === 'templates') { - args.push(dataRecord.method.pluginId); + args.push(templates); continue; + } args.push(context[param]); @@ -875,10 +880,8 @@ } else { if (key === 'htmlparser') { - // Debug http2 state. - - allResults.h2 = r.data.htmlparser.request.response.h2; + allResults.h2 = r.data.htmlparser.h2; } // First data has priority, do not override it. @@ -953,8 +956,31 @@ // After new context received - launch link post plugins. + var iterationPluginContexts = {}; + + var hasData = false; + for(var i = 0; i < result.length && !hasData; i++) { + var r = result[i]; + if (r.data && !r.error && r.method.name === 'getLink' || r.method.name === 'getLinks' || r.method.name === 'prepareLink') { + var links; + if (r.method.name === 'prepareLink') { + links = r.data && r.data.addLink; + } else { + links = r.data; + } + if (links) { + hasData = true;; + } + } + } + + if (hasData) { + runPostPluginsIterationCall('startIteration', iterationPluginContexts); + } + for(var i = 0; i < result.length; i++) { + var r = result[i]; if (r.data && !r.error && r.method.name === 'getLink' || r.method.name === 'getLinks' || r.method.name === 'prepareLink') { @@ -977,20 +1003,18 @@ links = _.compact(links); - var iterationPluginContexts = {}; - - runPostPluginsIterationCall('startIteration', iterationPluginContexts); - for(var j = 0; j < links.length; j++) { var link = links[j]; allResults.links.push(link); runPostPlugins(link, r, usedMethods, context, pluginsContexts, iterationPluginContexts, options, asyncMethodCb); } - - runPostPluginsIterationCall('finishIteration', iterationPluginContexts); } } + if (hasData) { + runPostPluginsIterationCall('finishIteration', iterationPluginContexts); + } + return hasNewData; } @@ -1160,7 +1184,7 @@ } // Sort links. - core.sortLinks(links); + sortLinks(links); var allData = result.allData; if (allData) { @@ -1186,7 +1210,7 @@ } } - core.sortLinks = function(links) { + export function sortLinks(links) { // Sort links in order of REL according to CONFIG.REL_GROUPS. function getRelIndex(rel) { @@ -1269,10 +1293,10 @@ var r = result[i]; var redirect = r.error && r.error[SYS_ERRORS.redirect]; if (typeof redirect === "string") { - sysUtils.log(' -- plugin redirect (by "' + r.method.pluginId + '")', redirect); + log(' -- plugin redirect (by "' + r.method.pluginId + '")', redirect); return redirect; } else if (redirect && typeof redirect !== "string") { - sysUtils.log(' -- skip plugin redirect, not string (by "' + r.method.pluginId + '")', redirect); + log(' -- skip plugin redirect, not string (by "' + r.method.pluginId + '")', redirect); } } } @@ -1291,17 +1315,17 @@ // error - if error is responseCode (from oembedError). var message = error.message || error; - sysUtils.log(' -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: message, uri: uri})); + log(' -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: message, uri: uri})); return error; } if (r.error && r.error[SYS_ERRORS.responseStatusCode]) { - sysUtils.log(' -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: r.error[SYS_ERRORS.responseStatusCode], uri: uri})); + log(' -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: r.error[SYS_ERRORS.responseStatusCode], uri: uri})); return r.error[SYS_ERRORS.responseStatusCode]; } if (r.error && r.error === SYS_ERRORS.timeout) { - sysUtils.log(' -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: SYS_ERRORS.timeout, uri: uri})); + log(' -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: SYS_ERRORS.timeout, uri: uri})); return SYS_ERRORS.timeout; } } @@ -1342,7 +1366,7 @@ var value = searchParamInObj(0, bits, options.providerOptions); - if (_.isArray(value)) { + if (Array.isArray(value)) { // ok. @@ -1395,7 +1419,7 @@ /* * Run plugins to collect all possible data. * */ - var run = core.run = function(uri, options, cb) { + export function run(uri, options, cb) { if (typeof options === 'function') { cb = options; @@ -1423,7 +1447,8 @@ }; if (!options.jar) { - options.jar = request.jar(); + // TODO: + //options.jar = request.jar(); } if ('disableCache' in options) { @@ -1489,7 +1514,8 @@ url: uri, cb: true, options: options, - request: requestWrapperWithParams + request: requestWrapperWithParams, + iframelyRun: run }, pluginsContexts = {}, @@ -1505,8 +1531,8 @@ aborted = false, abortCurrentRequest = function() { - if (context.htmlparser) { - utils.abortRequest(context.htmlparser.request); + if (context.htmlparser && context.htmlparser.abortController) { + context.htmlparser.abortController.abort(); } }, @@ -1638,13 +1664,13 @@ // If no data from domain plugins - try fallback to generic plugins. if (!options.mixAllWithDomainPlugin && isDomainPluginsMode && !resultsHasDomainData(requiredPlugins, allResults.allData)) { - sysUtils.log(' -- fallback from domain to generic', usedDomains, uri); + log(' -- fallback from domain to generic', usedDomains, uri); if (options.skipFallbackNotification) { delete options.skipFallbackNotification; } else { fallbackInfo = { - usedDomainsList: _.keys(usedDomains), + usedDomainsList: Object.keys(usedDomains), uri: uri }; } @@ -1692,13 +1718,13 @@ asyncMethodCb('initial'); }; - exports.getPluginData = function(uri, param, getWhitelistRecord, cb) { + export function getPluginData(uri, param, getWhitelistRecord, cb) { run(uri, { fetchParam: param, getWhitelistRecord: getWhitelistRecord }, cb); }; - exports.getOembed = oembedUtils.getOembed; + var getOembed = oembedUtils.getOembed; -})(exports); + export { getOembed }; diff --git a/lib/fetch.js b/lib/fetch.js new file mode 100644 index 000000000..ab24f36f9 --- /dev/null +++ b/lib/fetch.js @@ -0,0 +1,224 @@ +import { URL } from 'url'; +import { h1NoCache, noCache, createUrl, AbortController, AbortError, FetchError } from '@adobe/helix-fetch'; +import log from '../logging.js'; + +const fetchKeepAlive = noCache({ + h1: { + keepAlive: true + }, + rejectUnauthorized: false // By default skip auth check for all. +}).fetch; + +const fetchH1KeepAlive = h1NoCache({ + h1: { + keepAlive: true + }, + rejectUnauthorized: false // By default skip auth check for all. +}).fetch; + +const fetchAuthorized = noCache().fetch; // `rejectUnauthorized: true` - by `fetch` default. +const fetchH1Authorized = h1NoCache().fetch; // `rejectUnauthorized: true` - by `fetch` default. + +const { fetch } = noCache({ + rejectUnauthorized: false // By default skip auth check for all. +}); + +const fetchH1 = h1NoCache({ + rejectUnauthorized: false // By default skip auth check for all. +}).fetch; + +function doFetch(fetch_func, h1_fetch_func, options) { + + const fetch_options = Object.assign({}, options); + + // Implement `qs` (get params). + var uri = options.qs ? createUrl(options.uri, options.qs) : options.uri; + // Remove hash part of url. + uri = uri.replace(/#.*/gi, ''); + + const abortController = new HostAbortController(uri); + + // Allow request abort before finish. + fetch_options.signal = abortController.signal; + // Implement `timeout`. + const timeoutTimerId = setTimeout(() => { + abortController.abort(); + }, options.timeout || CONFIG.RESPONSE_TIMEOUT); + + const a_fetch_func = options.disable_http2 ? h1_fetch_func: fetch_func; + return new Promise((resolve, reject) => { + a_fetch_func(uri, fetch_options) + .then(response => { + var stream = response.body; + stream.on('end', () => { + clearTimeout(timeoutTimerId); + }); + abortController.onResponse(stream); + stream.status = response.status; + stream.headers = response.headers.plain(); + stream.abortController = abortController; + stream.h2 = response.httpVersion === '2.0'; + resolve(stream); + }) + .catch(error => { + clearTimeout(timeoutTimerId); + if (!options.disable_http2 && error.code && /^ERR_HTTP2/.test(error.code)) { + log(' -- doFetch http2 error', error.code, uri); + resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true}))); + } else { + if (!options.disable_http2 && error.code && error instanceof FetchError && error.code === 'ABORT_ERR') { + // Special case, when shared session request aborted by htmlparser logic. + /** + * https://polldaddy.com/poll/7451882/?s=twitter + * https://app.everviz.com/show/O0Cy7Dyt + */ + log(' -- doFetch h2 aborted error', uri); + resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true}))); + } else { + if (error instanceof AbortError) { + // `AbortError` before `response` occurs only on timeout. + error = 'timeout'; + } + reject(error); + } + } + }); + }); +} + +export function fetchStreamKeepAlive(options) { + return doFetch(fetchKeepAlive, fetchH1KeepAlive, options); +} + +export function fetchStream(options) { + return doFetch(fetch, fetchH1, options); +}; + +export function fetchStreamAuthorized(options) { + return doFetch(fetchAuthorized, fetchH1Authorized, options); +}; + +export function fetchData(options) { + var json = options.json; + delete options.json; + var res; + const fetch_options = Object.assign({}, options); + const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri; + + const abortController = new HostAbortController(uri); + + // Allow request abort before finish. + fetch_options.signal = abortController.signal; + // Implement `timeout`. + const timeoutTimerId = setTimeout(() => { + abortController.abort(); + }, options.timeout || CONFIG.RESPONSE_TIMEOUT); + + const a_fetch_func = options.disable_http2 ? fetchH1: fetch; + return new Promise((resolve, reject) => { + a_fetch_func(uri, fetch_options) + .then(response => { + var stream = response.body; + // TODO: looks like HEAD request has no END event. + stream.on('end', () => { + clearTimeout(timeoutTimerId); + }); + abortController.onResponse(stream); + res = response; + if (json !== false) { + // If `json` not forbidden, read `content-type`. + json = json || (response.headers.get('content-type').indexOf('application/json') > -1); + } + if (json) { + return response.json(); + } else { + return response.text(); + } + }) + .then(data => { + resolve({ + status: res.status, + headers: res.headers.plain(), + data: data + }); + }) + .catch((error) => { + clearTimeout(timeoutTimerId); + reject(error); + }); + }); +}; + +const hostsCache = {}; + +function addController(ctrl) { + hostsCache[ctrl.host] = hostsCache[ctrl.host] || []; + hostsCache[ctrl.host].push(ctrl); +} + +function tryAbortHost(host) { + var controllers = hostsCache[host]; + if (controllers) { + const hasWaitingRequests = controllers.some(ctrl => ctrl.waiting); + // If all aborted or finished. + if (!hasWaitingRequests) { + while (controllers.length) { + let ctrl = controllers.pop(); + ctrl.forceAbort(); + } + } + } +} + +function removeController(ctrl) { + var controllers = hostsCache[ctrl.host]; + if (controllers) { + const idx = controllers.indexOf(ctrl); + controllers.splice(idx, 1); + } +} + +class HostAbortController { + + constructor(url) { + this.aborted = false; + this.responded = false; + this.url = url; + const parsedUrl = new URL(url); + this.host = parsedUrl.protocol + parsedUrl.hostname; + this.abortController = new AbortController(); + addController(this); + } + + onResponse(stream) { + this.stream = stream; + if (this.aborted) { + stream.pause(); + } + stream.on('end', () => { + this.finished = true; + removeController(this); + tryAbortHost(this.host); + }); + } + + abort() { + this.stream && this.stream.pause(); + this.aborted = true; + tryAbortHost(this.host); + } + + forceAbort() { + if (this.aborted && !this.finished) { + this.abortController.abort(); + } + } + + get waiting() { + return !(this.aborted || this.finished); + } + + get signal() { + return this.abortController.signal; + } +} \ No newline at end of file diff --git a/lib/html-utils.js b/lib/html-utils.js index 578ac295c..b2a73b107 100644 --- a/lib/html-utils.js +++ b/lib/html-utils.js @@ -1,7 +1,7 @@ -(function() { - - var $ = require('cheerio'); - var _ = require('underscore'); + import cheerio from 'cheerio'; + + import * as _ from 'underscore'; + import CONFIG from '../config.loader.js'; var defaultPaddingBottom = 100 / CONFIG.DEFAULT_ASPECT_RATIO; @@ -30,7 +30,7 @@ } } - var $container = $('<div>') + var $container = cheerio('<div>') .append($element); if (aspectWrapperClass) { @@ -45,7 +45,7 @@ var hasMaxWidth = media && (media["max-width"] || media["min-width"] || media["width"] || verticalAspect); if (hasMaxWidth || forceWidthLimitContainer) { - $widthLimitContainer = $('<div>') + $widthLimitContainer = cheerio('<div>') .append($container); } @@ -166,7 +166,7 @@ && data.href; }, generate: function(data) { - var $img = $('<img>') + var $img = cheerio('<img>') .attr('src', data.href); if (data.title) { $img @@ -190,7 +190,7 @@ var givf = data.rel.indexOf('gifv') > -1; var autoplay = data.rel.indexOf('autoplay') > -1 || givf; - var $video = $('<video' + (givf ? ' loop muted webkit-playsinline' : ' controls') + (autoplay ? ' autoplay' : '') + '>Your browser does not support HTML5 video.</video>'); + var $video = cheerio('<video' + (givf ? ' loop muted webkit-playsinline' : ' controls') + (autoplay ? ' autoplay' : '') + '>Your browser does not support HTML5 video.</video>'); if (iframelyData && iframelyData.links) { @@ -280,7 +280,7 @@ }, generate: function(data, options) { - var $iframe = $('<iframe>') + var $iframe = cheerio('<iframe>') .attr('src', data.href) .css('border', '0') .attr('allowfullscreen', ''); @@ -330,17 +330,17 @@ } }; - exports.generateElementWrapperHtml = function(element, link, options) { + export function generateElementWrapperHtml(element, link, options) { if (typeof element === 'string') { - element = $(element); + element = cheerio(element); } var $el = wrapContainer(element, link, options); - return $('<div>').append($el).html(); + return cheerio('<div>').append($el).html(); }; - exports.generateLinkElementHtml = function(link, options) { + export function generateLinkElementHtml(link, options) { var $el = generateLinkElement(link, options); if (_.isString($el)) { return $el; @@ -348,7 +348,7 @@ if (options && options.canonical && link.href !== options.canonical) { $el.attr('data-embed-canonical', options.canonical); } - return $('<div>').append($el).html(); + return cheerio('<div>').append($el).html(); } else { return ''; } @@ -394,7 +394,7 @@ return []; }; - exports.filterLinksByRel = function(rel, links, options) { + export function filterLinksByRel(rel, links, options) { var options = options || {}; @@ -547,7 +547,7 @@ // Good link selection. - var sortLinks = exports.sortLinks = function(links, autoplay_first, horizontal_first, mp4_first) { + export function sortLinks(links, autoplay_first, horizontal_first, mp4_first) { var options = { autoplay_first: autoplay_first, @@ -676,7 +676,7 @@ }); } - exports.findMainLink = function(iframely_data, options) { + export function findMainLink(iframely_data, options) { var selectedLink; @@ -702,6 +702,4 @@ } return selectedLink; - }; - -})(); \ No newline at end of file + }; \ No newline at end of file diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js index 0859149c1..0ce66a98e 100644 --- a/lib/loader/pluginLoader.js +++ b/lib/loader/pluginLoader.js @@ -1,15 +1,14 @@ -(function(pluginLoader) { - var fs = require('fs'), - path = require('path'), - _ = require('underscore'), - async = require('async'), - url = require('url'); + import * as fs from 'fs'; + import * as path from 'path'; + import * as node_jslint from 'jslint'; + import { readFile } from 'fs/promises'; + var JSLINT = node_jslint.load('latest'); + import * as utils from './utils.js'; - var node_jslint = require('jslint'), - JSLINT = node_jslint.load('latest'); - - var utils = require('./utils'); + import CONFIG from '../../config.loader.js'; + // Global CONFIG used by plugins loaded during module import. + global.CONFIG = CONFIG; var PLUGIN_METHODS = utils.PLUGIN_METHODS, PLUGINS_FIELDS = utils.PLUGIN_FIELDS, @@ -17,15 +16,25 @@ POST_PLUGIN_DEFAULT_PARAMS = utils.POST_PLUGIN_DEFAULT_PARAMS, DEFAULT_PARAMS = utils.DEFAULT_PARAMS; - var plugins = pluginLoader._plugins = {}, - providedParamsDict = pluginLoader._providedParamsDict = {}, - usedParamsDict = pluginLoader._usedParamsDict = {}, - templates = pluginLoader._templates = {}, + + const plugins = {}, + providedParamsDict = {}, + usedParamsDict = {}, + templates = {}, metaMappings = {}, - pluginsList = pluginLoader._pluginsList = [], - postPluginsList = pluginLoader._postPluginsList = [], + pluginsList = [], + postPluginsList = [], pluginsByDomain = {}; + export { + plugins as _plugins, + providedParamsDict as _providedParamsDict, + usedParamsDict as _usedParamsDict, + templates as _templates, + pluginsList as _pluginsList, + postPluginsList as _postPluginsList + }; + /* * =================== * Loading plugins @@ -134,7 +143,7 @@ return methods; } - function loadPluginFile(pluginPath) { + async function loadPluginFile(pluginPath) { var bits = pluginPath.split(path.sep); @@ -142,24 +151,33 @@ var plugin; var pluginDeclaration = {}; + var notPlugin = false; + + var pluginLoadTimeout = setTimeout(() => { + console.error("-- Error: Timeout loading plugin " + pluginPath + ". Maybe recurring dependency."); + }, 1000); // Load plugin. try { - plugin = require(pluginPath); + plugin = await import(pluginPath); + clearTimeout(pluginLoadTimeout); + if (plugin.notPlugin) { + notPlugin = true; + } + plugin = plugin && plugin.default; } catch(ex) { console.error("Error loading plugin", pluginPath, ex); + clearTimeout(pluginLoadTimeout); return; } - if (plugin.notPlugin) { + if (notPlugin || plugin.notPlugin) { // Skip utils modules. return; } // Check if have required method. - var hasSign = _.some(PLUGINS_FIELDS, function(sign) { - return sign in plugin; - }); + var hasSign = PLUGINS_FIELDS.some(sign => sign in plugin); if (!hasSign) { console.warn("No plugin methods in " + pluginPath + ". Add exports.notPlugin = true; to skip this warning."); return; @@ -208,7 +226,7 @@ pluginDeclaration.re = [plugin.re]; } else if (plugin.re instanceof Array) { - if (!_.every(plugin.re, function(re) { return re instanceof RegExp; })) { + if (!plugin.re.every(re => re instanceof RegExp)) { console.error('Invalid RegExp in re of', pluginPath); return; } @@ -327,11 +345,12 @@ for(var i = 0; i < mixins.length; i++) { var mixin = mixins[i]; + if (plugins[mixin]) { + var m = plugins[mixin].getPluginLastModifiedDate(); - var m = plugins[mixin].getPluginLastModifiedDate(); - - if (m > modified) { - modified = m; + if (m > modified) { + modified = m; + } } } } @@ -339,36 +358,36 @@ return modified; } - function loadPluginDir(pluginPath) { + async function loadPluginDir(pluginPath) { // Scan plugin dir. var plugins = fs.readdirSync(pluginPath); - plugins.forEach(function(plugin_name) { + for (const plugin_name of plugins) { var plugin = path.resolve(pluginPath, plugin_name); var stats = fs.statSync(plugin); if (stats.isFile()) { - loadPluginFile(plugin); + await loadPluginFile(plugin); } - }); + }; } - function scanAllPluginsDir(modulePluginsPath) { + async function scanAllPluginsDir(modulePluginsPath) { // Scan mudule plugins. var plugins = fs.readdirSync(modulePluginsPath); - plugins.forEach(function(plugin_name) { + for (const plugin_name of plugins) { var plugin = path.resolve(modulePluginsPath, plugin_name); var stats = fs.statSync(plugin); if (stats.isFile()) { - loadPluginFile(plugin); + await loadPluginFile(plugin); } if (stats.isDirectory()) { - loadPluginDir(plugin); + await loadPluginDir(plugin); } - }); + }; } function extractDomain(uri) { @@ -406,7 +425,7 @@ return patterns; } - pluginLoader.findDomainPlugin = function(uri) { + export function findDomainPlugin(uri) { var patterns = extractDomainPatterns(uri); var record, i = 0; @@ -418,7 +437,7 @@ return record; }; - function scanModulesForPlugins() { + async function scanModulesForPlugins() { // Scan node_modules dir. var modulesRootPath = path.resolve('node_modules'); @@ -426,7 +445,7 @@ modules_listing.push(path.resolve('.')); - modules_listing.forEach(function(modulePath) { + for (const modulePath of modules_listing) { var modulePackagePath = path.resolve(modulePath, 'package.json'); @@ -434,43 +453,45 @@ // Scan plugins. - var moduleInfo = require(modulePackagePath); + + const moduleInfo = JSON.parse(await readFile(new URL(modulePackagePath, import.meta.url))); + if (!moduleInfo["iframely-proxy-plugins"]) { - return; + continue; } - function loadPlugins() { + async function loadPlugins() { var bits = Array.prototype.slice.call(arguments, 0); bits.splice(0, 0, modulePath); var modulePluginsPath = path.resolve.apply(path.resolve, bits); if (fs.existsSync(modulePluginsPath)) { - scanAllPluginsDir(modulePluginsPath); + await scanAllPluginsDir(modulePluginsPath); } } - loadPlugins('plugins', 'domains'); - loadPlugins('plugins', 'custom'); - loadPlugins('plugins', 'links'); - loadPlugins('plugins', 'meta'); - loadPlugins('plugins', 'templates'); + await loadPlugins('plugins', 'domains'); + await loadPlugins('plugins', 'custom'); + await loadPlugins('plugins', 'links'); + await loadPlugins('plugins', 'meta'); + await loadPlugins('plugins', 'templates'); // TODO: if has multiple modules_listing - CUSTOM_PLUGINS_PATH will be loaded multiple times. if (CONFIG.CUSTOM_PLUGINS_PATH) { if (fs.existsSync(CONFIG.CUSTOM_PLUGINS_PATH)) { - loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'domains'); - loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom'); - loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links'); - loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta'); - loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'domains'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta'); + await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates'); } else { console.warn('Custom plugin folder "' + CONFIG.CUSTOM_PLUGINS_PATH + '" not found.'); } } - loadPlugins('lib', 'plugins', 'system'); - loadPlugins('lib', 'plugins', 'validators'); + await loadPlugins('lib', 'plugins', 'system'); + await loadPlugins('lib', 'plugins', 'validators'); } - }); + } validateMixins(); validateDependencies(); @@ -542,13 +563,12 @@ } }); - var keys = _.keys(metaMappingsNotSorted); + var keys = Object.keys(metaMappingsNotSorted); keys.sort(); keys.forEach(function(key) { metaMappings[key] = metaMappingsNotSorted[key]; }); } - scanModulesForPlugins(); + await scanModulesForPlugins(); -})(exports); \ No newline at end of file diff --git a/lib/loader/utils.js b/lib/loader/utils.js index d8c1d8ef8..178d06726 100644 --- a/lib/loader/utils.js +++ b/lib/loader/utils.js @@ -1,24 +1,24 @@ -(function(utils) { - // TODO: not used anywhere. - utils.DEFAULT_PARAMS = [ + export const DEFAULT_PARAMS = [ "url", "urlMatch", "cb", "options", "request", "whitelistRecord", + "iframelyRun", "__readabilityEnabled" ]; - utils.POST_PLUGIN_DEFAULT_PARAMS = [ + export const POST_PLUGIN_DEFAULT_PARAMS = [ "link", "pluginContext", - "pluginId", + "plugin", + "templates", "iterationPluginContext" ]; - utils.PLUGIN_FIELDS_TYPE_DICT = { + export const PLUGIN_FIELDS_TYPE_DICT = { "getLink": Function, "getLinks": Function, "getData": Function, @@ -26,7 +26,7 @@ "mixins": Array }; - utils.PLUGIN_METHODS = [ + export const PLUGIN_METHODS = [ "getLink", "getLinks", "getData", @@ -34,8 +34,6 @@ "prepareLink" ]; - utils.PLUGIN_FIELDS = utils.PLUGIN_METHODS.concat([ + export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([ "mixins" - ]); - -})(exports); \ No newline at end of file + ]); \ No newline at end of file diff --git a/lib/oembed.js b/lib/oembed.js index 60b74f540..5c2c31d75 100644 --- a/lib/oembed.js +++ b/lib/oembed.js @@ -1,8 +1,7 @@ -var _ = require('underscore'); +import * as _ from 'underscore'; +import * as htmlUtils from './html-utils.js'; -var htmlUtils = require('./html-utils'); - -exports.getOembed = function(uri, data, options) { +export function getOembed(uri, data, options) { if (!data) { return; diff --git a/lib/plugins/system/cheerio.js b/lib/plugins/system/cheerio.js index a547d5848..d5832287e 100644 --- a/lib/plugins/system/cheerio.js +++ b/lib/plugins/system/cheerio.js @@ -1,8 +1,8 @@ -var cheerio = require('cheerio'); -var htmlparser2 = require("htmlparser2"); +import cheerio from 'cheerio'; +import htmlparser2 from "htmlparser2"; var DomHandler = htmlparser2.DomHandler; -module.exports = { +export default { provides: 'self', diff --git a/lib/plugins/system/decode.js b/lib/plugins/system/decode.js index 1075cd4bd..e89f9e537 100644 --- a/lib/plugins/system/decode.js +++ b/lib/plugins/system/decode.js @@ -1,6 +1,6 @@ -var utils = require('../../utils'); +import * as utils from '../../utils.js'; -module.exports = { +export default { provides: 'self', diff --git a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js index 0be2fde4c..efe915b80 100644 --- a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js +++ b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js @@ -5,39 +5,34 @@ * * */ -function CollectingHandlerForMutliTarget(cbsArray){ +export function CollectingHandlerForMutliTarget(cbsArray){ this._cbsArray = cbsArray || []; this.events = []; } -var EVENTS = require("htmlparser2").EVENTS; - -Object.keys(EVENTS).forEach(function(name) { - - if (EVENTS[name] === 0) { - - name = "on" + name; - CollectingHandlerForMutliTarget.prototype[name] = function() { - this.emitCb([name]); - }; - - } else if (EVENTS[name] === 1) { - - name = "on" + name; - CollectingHandlerForMutliTarget.prototype[name] = function(a) { - this.emitCb([name, a]); - }; - - } else if (EVENTS[name] === 2) { - - name = "on" + name; - CollectingHandlerForMutliTarget.prototype[name] = function(a, b) { - this.emitCb([name, a, b]); - }; - - } else { - throw Error("wrong number of arguments"); - } +// Got from `export interface Handler {` (Parser.d.ts). +const EVENTS = [ + 'onparserinit', + 'onreset', + 'onend', + 'onerror', + 'onclosetag', + 'onopentagname', + 'onattribute', + 'onopentag', + 'ontext', + 'oncomment', + 'oncdatastart', + 'oncdataend', + 'oncommentend', + 'onprocessinginstruction', +]; + +EVENTS.forEach(function(name) { + CollectingHandlerForMutliTarget.prototype[name] = function() { + let args = [name, ...arguments]; + this.emitCb(args); + }; }); CollectingHandlerForMutliTarget.prototype.addHandler = function(cbs) { @@ -94,20 +89,9 @@ CollectingHandlerForMutliTarget.prototype._onreset = function(cbs) { CollectingHandlerForMutliTarget.prototype.callCb = function(event, cbs) { function cb(cbs) { - - if (cbs[event[0]]) { - - var num = event.length; - - if (num === 1) { - cbs[event[0]](); - - } else if (num === 2) { - cbs[event[0]](event[1]); - - } else { - cbs[event[0]](event[1], event[2]); - } + const name = event[0]; + if (cbs[name]) { + cbs[name].apply(cbs, event.slice(1)); } } @@ -144,6 +128,4 @@ CollectingHandlerForMutliTarget.prototype._emitEventsFor = function(cbs) { } }; -module.exports = CollectingHandlerForMutliTarget; - -module.exports.notPlugin = true; \ No newline at end of file +export const notPlugin = true; \ No newline at end of file diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index eb774da80..31c1c91d5 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -1,17 +1,14 @@ -var _ = require('underscore'); -var urlLib = require('url'); -var htmlparser2 = require('htmlparser2'); -var Parser = htmlparser2.Parser; - -var cache = require('../../../cache'); -var utils = require('../../../utils'); -var libUtils = require('../../../utils'); -var metaUtils = require('../meta/utils'); +import * as _ from 'underscore'; +import * as urlLib from 'url'; +import { Parser } from 'htmlparser2'; +import { cache } from '../../../cache.js'; +import * as utils from '../../../utils.js'; +import * as libUtils from '../../../utils.js'; +import * as metaUtils from '../meta/utils.js'; var getUrlFunctional = utils.getUrlFunctional; +import { CollectingHandlerForMutliTarget } from './CollectingHandlerForMutliTarget.js'; -var CollectingHandlerForMutliTarget = require('./CollectingHandlerForMutliTarget'); - -module.exports = { +export default { provides: [ 'self', @@ -21,8 +18,6 @@ module.exports = { getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, cb) { - var request; - getUrlFunctional(url, _.extend({}, options, { followRedirect: !!options.followHTTPRedirect }), { @@ -37,19 +32,17 @@ module.exports = { }); } }, - onRequest: function(req) { - request = req; - }, onResponse: function(resp) { + function cacheAndRespond(error, data, preventCache) { var skip_cache = preventCache; - if (CONFIG.TEMP_HTTP_ERROR_CODES.indexOf(resp.statusCode) > -1) { + if (CONFIG.TEMP_HTTP_ERROR_CODES.indexOf(resp.status) > -1) { // Skip caching TEMP_HTTP_ERROR_CODES. skip_cache = true; } - if (resp.statusCode >= 500 && resp.statusCode <= 599) { + if (resp.status >= 500 && resp.status <= 599) { // Skip caching 5xx errors. skip_cache = true; } @@ -88,43 +81,44 @@ module.exports = { } } - if (resp.statusCode >= 300 && resp.statusCode < 400 && resp.headers.location) { - utils.abortRequest(request); - var redirectUrl = urlLib.resolve(url, resp.headers.location); + const headers = resp.headers; + const abortController = resp.abortController; + + if (resp.status >= 300 && resp.status< 400 && headers.location) { + var redirectUrl = urlLib.resolve(url, headers.location); // Prevent cache self redirect. Some sites changes cookies and stops redirect loop (e.g. https://miro.com/app/live-embed/o9J_lBwNMhI=/?embedAutoplay=true) var preventCache = redirectUrl === url; if (!options.exposeStatusCode) { return cacheAndRespond({ - redirect: resp.headers.location + redirect: headers.location }, null, preventCache); } else { return cacheAndRespond(null, { - headers: resp.headers + headers: headers }); } } - if (resp.statusCode !== 200) { - utils.abortRequest(request); + if (resp.status !== 200) { if (!!options.exposeStatusCode) { return cacheAndRespond(null, { - __statusCode: resp.statusCode, - headers: resp.headers + __statusCode: resp.status, + headers: headers }); } else { return cacheAndRespond({ - responseStatusCode: resp.statusCode + responseStatusCode: resp.status }); } } - if('content-type' in resp.headers && !/text\/html|application\/xhtml\+xml/gi.test(resp.headers['content-type'])){ - utils.abortRequest(request); + if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){ + abortController.abort(); return cacheAndRespond(null, { __nonHtmlContentData: { - type: resp.headers['content-type'], - content_length: resp.headers['content-length'], - 'set-cookie': resp.headers['set-cookie'] + type: headers['content-type'], + content_length: headers['content-length'], + 'set-cookie': headers['set-cookie'] } }); } @@ -132,19 +126,22 @@ module.exports = { // Init htmlparser handler. var handler = new CollectingHandlerForMutliTarget(); handler.onNoHandlers = function() { - if (!request.response.isPaused()) { - request.response.pause(); + if (!resp.isPaused()) { + resp.pause(); } }; handler.onFirstHandler = function() { - if (request.response.isPaused()) { - request.response.resume(); + if (resp.isPaused()) { + resp.resume(); } }; var parser = new Parser(handler, { - lowerCaseTags: true + lowerCaseTags: true, + decodeEntities: false // Fixes decoding html characters like   to UTF-8, we have own decoders. }); - handler.request = request; + handler.headers = headers; + handler.abortController = abortController; + handler.h2 = resp.h2; // Do before resume? cb(null, { diff --git a/lib/plugins/system/htmlparser/nonHtmlContentData.js b/lib/plugins/system/htmlparser/nonHtmlContentData.js index 59623f5cf..efcc69793 100644 --- a/lib/plugins/system/htmlparser/nonHtmlContentData.js +++ b/lib/plugins/system/htmlparser/nonHtmlContentData.js @@ -1,6 +1,6 @@ -var sysUtils = require('../../../../logging'); +import log from '../../../../logging.js'; -module.exports = { +export default { provides: [ // Run for all who requests htmlparser or meta. @@ -15,7 +15,7 @@ module.exports = { // HEADS UP: do not ever remove the below check for 'javascript' or 'flash' in content type // if left allowed, it'll make apps vulnerable for XSS attacks as such files will be rendered as regular embeds if (/javascript|flash|application\/json|text\/xml|application\/xml/i.test(nonHtmlContentType)) { - sysUtils.log(' -- Non html content type: "' + nonHtmlContentType + '" for ' + url); + log(' -- Non html content type: "' + nonHtmlContentType + '" for ' + url); return cb({ responseStatusCode: 415 }); diff --git a/lib/plugins/system/ld.js b/lib/plugins/system/ld.js index 320d7b1bb..1b92f1ed0 100644 --- a/lib/plugins/system/ld.js +++ b/lib/plugins/system/ld.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: 'self', diff --git a/lib/plugins/system/meta/HTMLMetaHandler.js b/lib/plugins/system/meta/HTMLMetaHandler.js index 20ccb5c28..abbb66886 100644 --- a/lib/plugins/system/meta/HTMLMetaHandler.js +++ b/lib/plugins/system/meta/HTMLMetaHandler.js @@ -1,10 +1,7 @@ -var decodeHTML5 = require('entities').decodeHTML5; -var _ = require('underscore'); -var url = require('url'); - -var utils = require('../../../utils'); - -var ldParser = require('./ld-json'); +import { decodeHTML5 } from 'entities'; +import * as url from 'url'; +import * as utils from '../../../utils.js'; +import { ldParser } from './ld-json.js'; var getCharset = utils.getCharset; var encodeText = utils.encodeText; @@ -25,7 +22,7 @@ var LINK_REL_ARRAY_VALUES = [ 'alternative' ]; -function HTMLMetaHandler(uri, contentType, callback) { +export function HTMLMetaHandler(uri, contentType, callback) { this._uri = uri; this._charset = getCharset(contentType); this._callback = callback; @@ -325,7 +322,7 @@ HTMLMetaHandler.prototype._finalMerge = function(tag) { function getSingleValue(parentName, obj) { var key = getDefaultKey(parentName); if (key in obj) { - if (_.keys(obj).length === 1) { + if (Object.keys(obj).length === 1) { return obj[key]; } } @@ -517,7 +514,7 @@ function merge(parentObj, props, value) { // New node. parentNode[currentNodeName] = value; - } else if (_.isArray(parentNode[currentNodeName])) { + } else if (Array.isArray(parentNode[currentNodeName])) { var append = false; @@ -584,6 +581,4 @@ function merge(parentObj, props, value) { } } -module.exports = HTMLMetaHandler; - -module.exports.notPlugin = true; +export const notPlugin = true; diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js index 2344d3178..968c5979a 100644 --- a/lib/plugins/system/meta/cachedMeta.js +++ b/lib/plugins/system/meta/cachedMeta.js @@ -1,10 +1,10 @@ -var urlLib = require('url'); -var cache = require('../../../cache'); -var sysUtils = require('../../../../logging'); -var utils = require('./utils'); -var libUtils = require('../../../utils'); +import * as urlLib from 'url'; +import { cache } from '../../../cache.js'; +import log from '../../../../logging.js'; +import * as utils from './utils.js'; +import * as libUtils from '../../../utils.js'; -module.exports = { +export default { provides: [ '__noCachedMeta', @@ -35,7 +35,7 @@ module.exports = { function callback(error, data) { if (error) { - sysUtils.log(' -- Error loading cached meta for: ' + url + '. ' + error); + log(' -- Error loading cached meta for: ' + url + '. ' + error); } function noCacheFound() { @@ -56,7 +56,7 @@ module.exports = { var record_age_sec = (new Date().getTime() / 1000) - data._sys_created_at; if (record_age_sec > ttl) { // Ignore cache older then requested ttl. - sysUtils.log(' -- Disable using old cache for: ' + url); + log(' -- Disable using old cache for: ' + url); return noCacheFound(); } } @@ -75,13 +75,13 @@ module.exports = { } // If data object has error attribute - return as error (e.g. redirect). - sysUtils.log(' -- Using cached htmlparser error for: ' + url); + log(' -- Using cached htmlparser error for: ' + url); cb(data.error); } else if (data.htmlparser) { - sysUtils.log(' -- Using cached htmlparser data for: ' + url); + log(' -- Using cached htmlparser data for: ' + url); cb(null, data.htmlparser); } else { - sysUtils.log(' -- Using cached meta for: ' + url); + log(' -- Using cached meta for: ' + url); cb(null, { meta: data, __hasCachedMeta: true, diff --git a/lib/plugins/system/meta/ld-json.js b/lib/plugins/system/meta/ld-json.js index a0e22e2fb..4f7c0ae7c 100644 --- a/lib/plugins/system/meta/ld-json.js +++ b/lib/plugins/system/meta/ld-json.js @@ -1,7 +1,7 @@ -const sysUtils = require('../../../../logging'); -const utils = require('../../../utils'); +import log from '../../../../logging.js'; +import * as utils from '../../../utils.js'; -module.exports = function(result, decode, uri) { +export function ldParser(result, decode, uri) { var ld = result["ld-json"]; delete result["ld-json"]; @@ -40,7 +40,7 @@ module.exports = function(result, decode, uri) { } catch (ex) { - sysUtils.log(' -- Error parsing ld-json', uri, ex.message); + log(' -- Error parsing ld-json', uri, ex.message); } } @@ -51,4 +51,4 @@ module.exports = function(result, decode, uri) { } }; -module.exports.notPlugin = true; \ No newline at end of file +export const notPlugin = true; \ No newline at end of file diff --git a/lib/plugins/system/meta/meta.js b/lib/plugins/system/meta/meta.js index 1a7c3a457..6f404abfb 100644 --- a/lib/plugins/system/meta/meta.js +++ b/lib/plugins/system/meta/meta.js @@ -1,11 +1,11 @@ -var HTMLMetaHandler = require('./HTMLMetaHandler'); -var cache = require('../../../cache'); -var sysUtils = require('../../../../logging'); -var libUtils = require('../../../utils'); -var iconv = require('iconv-lite'); -var utils = require('./utils'); +import { HTMLMetaHandler } from './HTMLMetaHandler.js'; +import { cache } from '../../../cache.js'; +import log from '../../../../logging.js'; +import * as libUtils from '../../../utils.js'; +import iconv from 'iconv-lite'; +import * as utils from './utils.js'; -module.exports = { +export default { provides: 'self', @@ -13,7 +13,7 @@ module.exports = { var metaHandler = new HTMLMetaHandler( url, - htmlparser.request.response.headers["content-type"], + htmlparser.headers["content-type"], function(error, meta) { htmlparser.removeHandler(metaHandler); @@ -27,7 +27,7 @@ module.exports = { // https://github.com/ashtuchkin/iconv-lite/issues/60 if (!iconv.encodingExists(meta.charset)) { - sysUtils.log(' -- Unsupported encoding: ' + meta.charset + ' in ' + url); + log(' -- Unsupported encoding: ' + meta.charset + ' in ' + url); return cb({ responseStatusCode: 415 }); @@ -46,7 +46,7 @@ module.exports = { }); if (options.refresh) { - sysUtils.log(' -- Refreshed meta cache for: ' + url); + log(' -- Refreshed meta cache for: ' + url); } } diff --git a/lib/plugins/system/meta/utils.js b/lib/plugins/system/meta/utils.js index 90305ab42..ff5c1fa70 100644 --- a/lib/plugins/system/meta/utils.js +++ b/lib/plugins/system/meta/utils.js @@ -1,4 +1,4 @@ -exports.getMetaCacheKey = function(url, whitelistRecord, options) { +export function getMetaCacheKey(url, whitelistRecord, options) { var meta_key = 'meta:' + url; @@ -15,4 +15,4 @@ exports.getMetaCacheKey = function(url, whitelistRecord, options) { return meta_key; }; -module.exports.notPlugin = true; \ No newline at end of file +export const notPlugin = true; \ No newline at end of file diff --git a/lib/plugins/system/oembed/autoDiscovery.js b/lib/plugins/system/oembed/autoDiscovery.js index 50bc9b54c..050963729 100644 --- a/lib/plugins/system/oembed/autoDiscovery.js +++ b/lib/plugins/system/oembed/autoDiscovery.js @@ -1,7 +1,7 @@ -const oembedUtils = require('./oembedUtils'); +import * as oembedUtils from './oembedUtils.js'; const RE = CONFIG.IGNORE_DOMAINS_RE || CONFIG.BLACKLIST_DOMAINS_RE; -module.exports = { +export default { provides: 'oembedLinks', diff --git a/lib/plugins/system/oembed/knownEndpoints.js b/lib/plugins/system/oembed/knownEndpoints.js index a55fdb82b..b767eca7c 100644 --- a/lib/plugins/system/oembed/knownEndpoints.js +++ b/lib/plugins/system/oembed/knownEndpoints.js @@ -1,6 +1,6 @@ -var oembedUtils = require('./oembedUtils'); +import * as oembedUtils from './oembedUtils.js'; -module.exports = { +export default { provides: ['oembedLinks', '__noOembedLinks'], diff --git a/lib/plugins/system/oembed/oembed.js b/lib/plugins/system/oembed/oembed.js index 8026aca8e..63369052b 100644 --- a/lib/plugins/system/oembed/oembed.js +++ b/lib/plugins/system/oembed/oembed.js @@ -1,9 +1,9 @@ -const oembedUtils = require('./oembedUtils'); -const cheerio = require('cheerio'); -const entities = require('entities'); +import * as oembedUtils from './oembedUtils.js'; +import cheerio from 'cheerio'; -const URL = require('url'); -const querystring = require('querystring'); +import * as entities from 'entities'; +import * as URL from 'url'; +import * as querystring from 'querystring'; const pxRe = /^([\d\.]+)(?:px)?$/; const percentRe = /^([\d\.]+)%$/; @@ -51,11 +51,20 @@ function _getOembedIframe(oembed) { oembed._iframe = fixOembedIframeAttributes($iframe[0].attribs); if (oembed._iframe && oembed._iframe.src) { - oembed._iframe.query = URL.parse(oembed._iframe.src, true).query; + var src = URL.parse(oembed._iframe.src, true); + oembed._iframe.host = src.host; + oembed._iframe.pathname = src.pathname; + oembed._iframe.path = src.path; + oembed._iframe.query = src.query; + oembed._iframe.placeholder = oembed.thumbnail_url; oembed._iframe.replaceQuerystring = function(params) { var qs = querystring.stringify({...oembed._iframe.query, ...params}); return oembed._iframe.src.replace(/\?.*$/, '') + (qs ? '?' + qs : ''); } + oembed._iframe.assignQuerystring = function(params) { + var qs = querystring.stringify(params); + return oembed._iframe.src.replace(/\?.*$/, '') + (qs ? '?' + qs : ''); + } } } else { oembed._iframe = null; @@ -80,7 +89,7 @@ function getOembedIframeAttr(oembed) { }; } -module.exports = { +export default { provides: ['self', 'oembedError'], diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js index 1c5030d3b..a59f9187b 100644 --- a/lib/plugins/system/oembed/oembedUtils.js +++ b/lib/plugins/system/oembed/oembedUtils.js @@ -1,17 +1,19 @@ -var sax = require('sax'); -var urlLib = require('url'); -var async = require('async'); - -var utils = require('../../../utils'); -var sysUtils = require('../../../../logging'); -var cache = require('../../../cache'); +import sax from 'sax'; +import * as urlLib from 'url'; +import * as async from 'async'; +import * as utils from '../../../utils.js'; +import log from '../../../../logging.js'; +import { cache } from '../../../cache.js'; +import { readFile } from 'fs/promises'; var getUrlFunctional = utils.getUrlFunctional; var getCharset = utils.getCharset; var encodeText = utils.encodeText; var lowerCaseKeys = utils.lowerCaseKeys; -exports.notPlugin = true; +export const notPlugin = true; + +const providers = JSON.parse(await readFile(new URL('./providers.json', import.meta.url))); /** * @private @@ -20,8 +22,6 @@ exports.notPlugin = true; * @return {String} The oembed uri */ function lookupStaticProviders(uri) { - var providers = require('./providers.json'); - var protocolMatch = uri.match(/^(https?:\/\/)/); if (!protocolMatch || /^https?:\/\/blog\./i.test(uri)) { return null; @@ -83,7 +83,7 @@ function lookupStaticProviders(uri) { return links; } -module.exports.findOembedLinks = function(uri, meta) { +export function findOembedLinks(uri, meta) { // Filter oembed from meta. // allow misspelled discovery links @@ -125,7 +125,7 @@ module.exports.findOembedLinks = function(uri, meta) { * @param {String} uri Full oEmbed endpoint plus URL and any needed format parameter. * @param {Function} callback Completion callback function. The callback gets two arguments (error, oembed) where oembed is json parsed oEmbed object. * */ -module.exports.getOembed = function(uri, options, callback) { + export function getOembed(uri, options, callback) { if (typeof options === 'function') { callback = options; @@ -181,7 +181,7 @@ module.exports.getOembed = function(uri, options, callback) { // Fallback to `request` format. Remove after 10.10.2020. var cachedError = (data && data.response && data.data && data.data.error) || (data && data.error); if (cachedError) { - sysUtils.log(' -- Oembed cache error: ' + uri, JSON.stringify(data)); + log(' -- Oembed cache error: ' + uri, JSON.stringify(data)); } if (data && !cachedError) { @@ -189,7 +189,7 @@ module.exports.getOembed = function(uri, options, callback) { // Fallback to `request` format. Remove after 10.10.2020. data = data.data; } - sysUtils.log(' -- Using cached oembed for: ' + uri); + log(' -- Using cached oembed for: ' + uri); return cb({good_data_from_cache: true}, data); } @@ -227,17 +227,15 @@ module.exports.getOembed = function(uri, options, callback) { }); } - getUrlFunctional(uri, { - disableHttp2: options && options.disableHttp2 - }, { + getUrlFunctional(uri, {}, { onError: cbWrapper, onResponse: function(res) { - if (res.statusCode == 200) { + if (res.status == 200) { parseResponse(res); } else if (options && options.parseErrorBody) { - parseResponse(res, res.statusCode); + parseResponse(res, res.status); } else { - cbWrapper(res.statusCode); + cbWrapper(res.status); } } }); @@ -354,7 +352,7 @@ function xmlStream2oembed(stream, callback) { // Decode only non UTF-8 because sax makes decoding. var oldValue = value; value = encodeText(charset, value); - sysUtils.log(' -- decode oembed xml (charset, in, out):', charset, oldValue, value); + log(' -- decode oembed xml (charset, in, out):', charset, oldValue, value); } if (prop.match(/(width|height)$/)) { diff --git a/lib/plugins/system/og.js b/lib/plugins/system/og.js index 21a24b7ed..abd626c9b 100644 --- a/lib/plugins/system/og.js +++ b/lib/plugins/system/og.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: 'self', diff --git a/lib/plugins/system/readability.js b/lib/plugins/system/readability.js index 226744e86..fdbd84d7e 100644 --- a/lib/plugins/system/readability.js +++ b/lib/plugins/system/readability.js @@ -1,6 +1,6 @@ -var Readability = require('readabilitySAX').Readability; +import { Readability } from 'readabilitySAX'; -module.exports = { +export default { provides: 'self', @@ -44,8 +44,6 @@ module.exports = { readability.onend = onEnd; htmlparser.addHandler(readability); - - htmlparser.request.response.on('onerror', cb); } }; \ No newline at end of file diff --git a/lib/plugins/system/twitter.js b/lib/plugins/system/twitter.js index 7d5b4e262..4c26e4ad0 100644 --- a/lib/plugins/system/twitter.js +++ b/lib/plugins/system/twitter.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: 'self', diff --git a/lib/plugins/validators/async/20_checkFavicon.js b/lib/plugins/validators/async/20_checkFavicon.js index 6ecce3aa0..6d095e783 100644 --- a/lib/plugins/validators/async/20_checkFavicon.js +++ b/lib/plugins/validators/async/20_checkFavicon.js @@ -1,10 +1,9 @@ -var urlLib = require('url'); -var _ = require('underscore'); +import * as urlLib from 'url'; +import * as _ from 'underscore'; +import * as utils from '../../../utils.js'; +import { cache } from '../../../cache.js'; -var utils = require('../../../utils'); -var cache = require('../../../cache'); - -module.exports = { +export default { startIteration: function(iterationPluginContext) { iterationPluginContext.multiCache = new cache.MultiCache(); diff --git a/lib/plugins/validators/async/21_checkContentType.js b/lib/plugins/validators/async/21_checkContentType.js index 166ca7d90..5d01b1097 100644 --- a/lib/plugins/validators/async/21_checkContentType.js +++ b/lib/plugins/validators/async/21_checkContentType.js @@ -1,10 +1,10 @@ -const utils = require('../../../utils'); -const urlLib = require('url'); -const multimedia = require('../html5_multimedia'); -const mediaPlugin = require('../media'); -const player_no_scrolling = require('../player_no_scrolling'); +import * as utils from '../../../utils.js'; +import * as urlLib from 'url'; +import multimedia from '../html5_multimedia.js'; +import mediaPlugin from '../media.js'; +import player_no_scrolling from '../player_no_scrolling.js'; -module.exports = { +export default { prepareLink: function(url, link, options, pluginContext, cb) { diff --git a/lib/plugins/validators/async/22_imageSize.js b/lib/plugins/validators/async/22_imageSize.js index 797f3d5ad..9ea36e8d0 100644 --- a/lib/plugins/validators/async/22_imageSize.js +++ b/lib/plugins/validators/async/22_imageSize.js @@ -1,9 +1,9 @@ -const utils = require('../../../utils'); -const cache = require('../../../cache'); -const mediaPlugin = require('../media'); +import * as utils from '../../../utils.js'; +import { cache } from '../../../cache.js'; +import mediaPlugin from '../media.js'; -module.exports = { +export default { startIteration: function(iterationPluginContext) { iterationPluginContext.multiCache = new cache.MultiCache(); diff --git a/lib/plugins/validators/html5_multimedia.js b/lib/plugins/validators/html5_multimedia.js index 747d9e41d..08ecc65e0 100644 --- a/lib/plugins/validators/html5_multimedia.js +++ b/lib/plugins/validators/html5_multimedia.js @@ -1,4 +1,4 @@ -module.exports = { +export default { prepareLink: function(link) { diff --git a/lib/plugins/validators/media.js b/lib/plugins/validators/media.js index 985e802b1..3880bde90 100644 --- a/lib/plugins/validators/media.js +++ b/lib/plugins/validators/media.js @@ -92,7 +92,7 @@ function moveMediaAttrs(link) { } } -module.exports = { +export default { notPlugin: true, diff --git a/lib/plugins/validators/player_no_scrolling.js b/lib/plugins/validators/player_no_scrolling.js index fb65a851f..691655d9e 100644 --- a/lib/plugins/validators/player_no_scrolling.js +++ b/lib/plugins/validators/player_no_scrolling.js @@ -1,4 +1,4 @@ -module.exports = { +export default { prepareLink: function(link) { diff --git a/lib/plugins/validators/sync/00_validateLink.js b/lib/plugins/validators/sync/00_validateLink.js index 5db62d447..f49dcfa26 100644 --- a/lib/plugins/validators/sync/00_validateLink.js +++ b/lib/plugins/validators/sync/00_validateLink.js @@ -1,6 +1,6 @@ -const urlLib = require('url'); +import * as urlLib from 'url'; -module.exports = { +export default { prepareLink: function(url, link) { diff --git a/lib/plugins/validators/sync/01_qa_rels.js b/lib/plugins/validators/sync/01_qa_rels.js index f204d8953..65e02a797 100644 --- a/lib/plugins/validators/sync/01_qa_rels.js +++ b/lib/plugins/validators/sync/01_qa_rels.js @@ -1,17 +1,12 @@ -var pluginLoader = require('../../../loader/pluginLoader'), - plugins = pluginLoader._plugins; +export default { -module.exports = { - - prepareLink: function(whitelistRecord, options, link, pluginId) { + prepareLink: function(whitelistRecord, options, link, plugin) { if (link.type === CONFIG.T.flash) { link.error = 'Adobe Flash Player is no longer supported'; return; } - var plugin = plugins[pluginId]; - if (plugin.domain || plugin.custom || (link.rel && (link.rel.indexOf(CONFIG.R.icon) > -1 || link.rel.indexOf(CONFIG.R.thumbnail) > -1))) { return; } diff --git a/lib/plugins/validators/sync/02_html5_multimedia.js b/lib/plugins/validators/sync/02_html5_multimedia.js index dbfb0bafc..d183dfc5b 100644 --- a/lib/plugins/validators/sync/02_html5_multimedia.js +++ b/lib/plugins/validators/sync/02_html5_multimedia.js @@ -1,6 +1,6 @@ -const multimedia = require('../html5_multimedia'); +import multimedia from '../html5_multimedia.js'; -module.exports = { +export default { prepareLink: function(link, options) { multimedia.prepareLink(link, options); diff --git a/lib/plugins/validators/sync/03_media.js b/lib/plugins/validators/sync/03_media.js index 37b5e0812..183cfae28 100644 --- a/lib/plugins/validators/sync/03_media.js +++ b/lib/plugins/validators/sync/03_media.js @@ -1,6 +1,6 @@ -var mediaPlugin = require('../media'); +import mediaPlugin from '../media.js'; -module.exports = { +export default { prepareLink: function(link, options) { diff --git a/lib/plugins/validators/sync/04_ssl_validator.js b/lib/plugins/validators/sync/04_ssl_validator.js index 3821fe267..49d46d51a 100644 --- a/lib/plugins/validators/sync/04_ssl_validator.js +++ b/lib/plugins/validators/sync/04_ssl_validator.js @@ -1,4 +1,4 @@ -module.exports = { +export default { prepareLink: function(link, url, options) { diff --git a/lib/plugins/validators/sync/05_camoImage.js b/lib/plugins/validators/sync/05_camoImage.js index 50db4e887..7bd1c8d31 100644 --- a/lib/plugins/validators/sync/05_camoImage.js +++ b/lib/plugins/validators/sync/05_camoImage.js @@ -1,8 +1,7 @@ /* Should be last async plugin to prevent size check over camo. */ - -var crypto = require('crypto'); +import * as crypto from 'crypto'; function isSSL(link) { return link.rel.indexOf('ssl') > -1; @@ -12,7 +11,7 @@ function isImage(link) { return link.type.match(/^image/); } -module.exports = { +export default { // Get from default CONFIG. Not supposed to be enabled by dynamic custom provider options. notPlugin: !CONFIG.providerOptions.camoProxy, diff --git a/lib/plugins/validators/sync/06_autoplay.js b/lib/plugins/validators/sync/06_autoplay.js index 2015e164f..2733bf16a 100644 --- a/lib/plugins/validators/sync/06_autoplay.js +++ b/lib/plugins/validators/sync/06_autoplay.js @@ -1,6 +1,6 @@ -var _ = require('underscore'); +import * as _ from 'underscore'; -module.exports = { +export default { prepareLink: function(link) { diff --git a/lib/plugins/validators/sync/07_resizable.js b/lib/plugins/validators/sync/07_resizable.js index 2f1a4f16e..14131b1bf 100644 --- a/lib/plugins/validators/sync/07_resizable.js +++ b/lib/plugins/validators/sync/07_resizable.js @@ -1,7 +1,7 @@ -const utils = require('../../../utils'); -const URL = require('url'); +import * as utils from '../../../utils.js'; +import * as URL from 'url'; -module.exports = { +export default { prepareLink: function(url, options, link) { diff --git a/lib/plugins/validators/sync/08_render.js b/lib/plugins/validators/sync/08_render.js index 9f985bc4e..41276c74b 100644 --- a/lib/plugins/validators/sync/08_render.js +++ b/lib/plugins/validators/sync/08_render.js @@ -1,15 +1,12 @@ -var ejs = require('ejs'), - fs = require('fs'); +import * as ejs from 'ejs'; +import * as fs from 'fs'; -var pluginLoader = require('../../../loader/pluginLoader'), - templates = pluginLoader._templates; - -module.exports = { - prepareLink: function(link, pluginId) { +export default { + prepareLink: function(link, plugin, templates) { if (!link.href && !link.html && (link.template || link.template_context)) { - var template = link.template || pluginId; + var template = link.template || plugin.id; if (!(template in templates)) { console.error("No template found: " + template); diff --git a/lib/plugins/validators/sync/09_app_hml5.js b/lib/plugins/validators/sync/09_app_hml5.js index 3bc64e21e..9ab459855 100644 --- a/lib/plugins/validators/sync/09_app_hml5.js +++ b/lib/plugins/validators/sync/09_app_hml5.js @@ -1,4 +1,4 @@ -module.exports = { +export default { prepareLink: function(link) { // Add 'html5' rel to 'app'. diff --git a/lib/plugins/validators/sync/10_video_hml5.js b/lib/plugins/validators/sync/10_video_hml5.js index 29e4920f8..0386bfea6 100644 --- a/lib/plugins/validators/sync/10_video_hml5.js +++ b/lib/plugins/validators/sync/10_video_hml5.js @@ -1,4 +1,4 @@ -module.exports = { +export default { prepareLink: function(link) { // Add 'html5' rel to 'video/*'. diff --git a/lib/plugins/validators/sync/11_image_type_by_ext.js b/lib/plugins/validators/sync/11_image_type_by_ext.js index 3555038e2..000ec43bc 100644 --- a/lib/plugins/validators/sync/11_image_type_by_ext.js +++ b/lib/plugins/validators/sync/11_image_type_by_ext.js @@ -1,4 +1,4 @@ -module.exports = { +export default { prepareLink: function(link) { diff --git a/lib/plugins/validators/sync/12_player_no_scrolling.js b/lib/plugins/validators/sync/12_player_no_scrolling.js index da8581ade..494f19d8a 100644 --- a/lib/plugins/validators/sync/12_player_no_scrolling.js +++ b/lib/plugins/validators/sync/12_player_no_scrolling.js @@ -1,6 +1,6 @@ -const player_no_scrolling = require('../player_no_scrolling'); +import player_no_scrolling from '../player_no_scrolling.js'; -module.exports = { +export default { prepareLink: function(link, options) { player_no_scrolling.prepareLink(link, options); diff --git a/lib/plugins/validators/sync/14_options.js b/lib/plugins/validators/sync/14_options.js index 680be6adc..175855bfe 100644 --- a/lib/plugins/validators/sync/14_options.js +++ b/lib/plugins/validators/sync/14_options.js @@ -1,4 +1,4 @@ -module.exports = { +export default { prepareLink: function(link) { diff --git a/lib/plugins/validators/sync/15_duplicateLink.js b/lib/plugins/validators/sync/15_duplicateLink.js index af00028b9..2587163d7 100644 --- a/lib/plugins/validators/sync/15_duplicateLink.js +++ b/lib/plugins/validators/sync/15_duplicateLink.js @@ -1,4 +1,4 @@ -var _ = require('underscore'); +import * as _ from 'underscore'; function findBestMedia(m1, m2) { @@ -57,7 +57,7 @@ function findBestMedia(m1, m2) { return (m1.width > m2.width || m1.height > m2.height) ? m1 : m2; } -module.exports = { +export default { prepareLink: function(link, pluginContext) { diff --git a/lib/request.js b/lib/request.js index 4b6a1c096..657c87bdd 100644 --- a/lib/request.js +++ b/lib/request.js @@ -1,10 +1,9 @@ -var request = require('request'); -var _ = require('underscore'); -var crypto = require('crypto'); -var cache = require('./cache'); -var utils = require('./utils'); +import * as crypto from 'crypto'; +import { cache } from './cache.js'; +import * as utils from './utils.js'; +import log from '../logging.js'; -var sysUtils = require('../logging'); +import { fetchData } from './fetch.js'; var hash = function(value) { return '"' + crypto.createHash('md5').update(value).digest("hex") + '"'; @@ -28,9 +27,23 @@ var hash = function(value) { * - callback - final callback will called with data received from `prepareResult`. * * */ -module.exports = function(options, iframely_options, callback) { - options = _.extend({}, options); +/* + +TOOD: + +options: + +jar: false, +limit: 1, + +oauth + +*/ + +export default function(options, iframely_options, callback) { + + options = Object.assign({}, options); if (typeof options.prepareResult !== 'function') { console.error('cached request call error: prepareResult not a function'); @@ -54,8 +67,6 @@ module.exports = function(options, iframely_options, callback) { delete options.new_cache_key; delete options.refresh; - options.gzip = true; - var prefix = CONFIG.API_REQUEST_CACHE_PREFIX ? (CONFIG.API_REQUEST_CACHE_PREFIX + ':') : ''; var lang = iframely_options.getProviderOptions('locale'); @@ -67,23 +78,26 @@ module.exports = function(options, iframely_options, callback) { function doRealRequest(secondAttempt) { - request(utils.prepareRequestOptions(options, iframely_options), function(error, response, data) { - + function finish(error, result) { if (error) { - var url = options.url || options.uri; - // Retry on ECONNRESET. if (!secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) { - sysUtils.log(' -- request.js ' + error.code + ' first', url); + log(' -- request.js ' + error.code + ' first', options.uri); process.nextTick(function() { doRealRequest(true); }); return; } else if (secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) { - sysUtils.log(' -- request.js ' + error.code + ' second', url); + log(' -- request.js ' + error.code + ' second', options.uri); } } + const response = result && { + statusCode: result.status, + headers: result.headers + }; + const data = result && result.data; + prepareResult(error, response, data, function(preparedError, preparedData) { var doCaching; @@ -122,7 +136,18 @@ module.exports = function(options, iframely_options, callback) { // Send prepared data up. callback(preparedError, preparedData); }); - }); + } + + if (!options.headers || !options.headers['User-Agent']) { + options.headers = options.headers || {}; + options.headers['User-Agent'] = CONFIG.USER_AGENT; + } + + fetchData(utils.prepareRequestOptions(options, iframely_options)) + .then(result => { + finish(null, result); + }) + .catch(finish); } if (refresh) { @@ -157,7 +182,7 @@ module.exports = function(options, iframely_options, callback) { } // Send cached data up. - prepareResult(null, _.extend(data.response, {fromRequestCache: true}), data.data, callback); + prepareResult(null, Object.assign(data.response, {fromRequestCache: true}), data.data, callback); if (new_cache_key) { cache.set('req:' + prefix + new_cache_key, data, { diff --git a/lib/utils.js b/lib/utils.js index 00be2f252..595148192 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,26 +1,16 @@ -var events = require('events'); -var request = require('request'); -var Http2Agent = require('./agent-http2').Http2Agent; -var zlib = require('zlib'); -var decompressStream = require('iltorb').decompressStream; -var iconv = require('iconv-lite'); -var async = require('async'); -var imagesize = require('probe-image-size'); -var _ = require('underscore'); - -var parseIsoDuration = require('parse-iso-duration'); -var entities = require('entities'); - -var cache = require('./cache'); -var htmlUtils = require('./html-utils'); - -var sysUtils = require('../logging'); - -var http2Agent = new Http2Agent(/* options */); - -if (!global.CONFIG) { - global.CONFIG = require('../config'); -} +import * as events from 'events'; +import { fetchStream, fetchStreamKeepAlive, fetchStreamAuthorized } from './fetch.js'; +import * as iconv from 'iconv-lite'; +import * as async from 'async'; +import imagesize from 'probe-image-size'; +import * as _ from 'underscore'; +import * as parseIsoDuration from 'parse-iso-duration'; +import * as entities from 'entities'; +import { cache } from './cache.js'; +import * as htmlUtils from './html-utils.js'; +import log from '../logging.js'; + +import CONFIG from '../config.loader.js'; function prepareEncodedUri(request_options, attr) { var url = request_options[attr]; @@ -47,7 +37,7 @@ function maxTTL(ttl) { return ttl; } -var getMaxCacheTTLOverride = exports.getMaxCacheTTLOverride = function(url, options) { +export function getMaxCacheTTLOverride(url, options) { var proxy = null; if (CONFIG.PROXY || (options && options.proxy)) { proxy = (options && options.proxy) || CONFIG.PROXY.find(p => { @@ -57,7 +47,7 @@ var getMaxCacheTTLOverride = exports.getMaxCacheTTLOverride = function(url, opti return proxy; }; -var getMaxCacheTTL = exports.getMaxCacheTTL = function(url, options, default_min_ttl) { +export function getMaxCacheTTL(url, options, default_min_ttl) { var proxy = getMaxCacheTTLOverride(url, options); var result = Math.max(fixTTL(options && options.cache_ttl), fixTTL(proxy && proxy.cache_ttl), fixTTL(default_min_ttl)); result = maxTTL(result); @@ -72,24 +62,25 @@ var getMaxCacheTTL = exports.getMaxCacheTTL = function(url, options, default_min return result; }; -var prepareRequestOptions = exports.prepareRequestOptions = function(request_options, options) { - - var url = request_options.uri || request_options.url; +export function prepareRequestOptions(request_options, options) { - var disableHttp2 = options && options.disableHttp2 || CONFIG.DISABLE_HTTP2; + if (request_options.url && !request_options.uri) { + request_options.uri = request_options.url; + } + var uri = request_options.uri; + delete request_options.url; if (CONFIG.PROXY || (options && options.proxy)) { var proxy = (options && options.proxy) || CONFIG.PROXY.find(p => { - return p && p.re && p.re.some(re => url.match(re)); + return p && p.re && p.re.some(re => uri.match(re)); }); if (proxy) { if (proxy.prerender && CONFIG.PRERENDER_URL) { - delete request_options.uri; - delete request_options.url; - // `url` used above for agent check. - url = CONFIG.PRERENDER_URL + encodeURIComponent(url); - request_options.uri = url; + request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); + + } else if (proxy.proxy && CONFIG.PROXY_URL) { + request_options.uri = CONFIG.PROXY_URL + encodeURIComponent(uri); } if (proxy.proxy_server) { request_options.proxy = proxy.proxy_server; @@ -106,62 +97,22 @@ var prepareRequestOptions = exports.prepareRequestOptions = function(request_opt _.extend(request_options, proxy.request_options); } if (proxy.disable_http2) { - disableHttp2 = true; + request_options.disable_http2 = true; } } } - if (!disableHttp2 - && !request_options.proxy - && /^https:/.test(url) - && (!request_options.method - || request_options.method === 'GET' - || request_options.method === 'HEAD')) { - // http2 only for ssl and without proxy. - request_options.agent = http2Agent; - } - var lang = options && options.getProviderOptions && options.getProviderOptions('locale', 'en-US'); if (lang) { request_options.headers = request_options.headers || {}; request_options.headers['Accept-Language'] = lang.replace('_', '-') + CONFIG.ACCEPT_LANGUAGE_SUFFIX; } - prepareEncodedUri(request_options, 'url'); prepareEncodedUri(request_options, 'uri'); return request_options; }; -var getUrlFunctional = exports.getUrlFunctional = function(url, options, callbacks) { - - getUrl(url, options) - .on('error', function(error, opts) { - - // Retry on ECONNRESET. Http2Agent will try with `http1` after this error. - if (!options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) { - sysUtils.log(' -- getUrl ' + error.code + ' first', opts, url); - process.nextTick(function() { - getUrlFunctional(url, _.extend({}, options, { - secondAttempt: true, - disableHttp2: true - }), callbacks); - }); - return; - } else if (options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) { - sysUtils.log(' -- getUrl ' + error.code + ' second', opts, url); - } - - callbacks.onError && callbacks.onError(error); - }) - .on('request', function(request) { - callbacks.onRequest && callbacks.onRequest(request); - }) - .on('response', function(response) { - callbacks.onResponse && callbacks.onResponse(response); - }); -} - /** * @public * Do HTTP GET request and handle redirects @@ -172,178 +123,109 @@ var getUrlFunctional = exports.getUrlFunctional = function(url, options, callbac * @param {Function} [callback] The completion callback function or events.EventEmitter object * @returns {events.EventEmitter} The emitter object which emit error or response event */ -var getUrl = exports.getUrl = function(url, options) { - - var req = new events.EventEmitter(); + export function getUrl(url, options, callbacks) { var options = options || {}; - // Store cookies between redirects and requests. - var jar = options.jar; - if (!jar) { - jar = request.jar(); + var redirect, follow; + if (options.followRedirect) { + redirect = 'follow'; + follow = options.maxRedirects || CONFIG.MAX_REDIRECTS; + } + if (options.followRedirect === false) { + redirect = 'manual'; + follow = 0; } - process.nextTick(function() { - try { - - var sslProtocol = /^(https:)?\/\//i.test(url); - - var request_options = prepareRequestOptions({ - uri: url, - method: 'GET', - headers: { - 'User-Agent': options.user_agent || CONFIG.USER_AGENT, - 'Connection': 'keep-alive', - 'Accept-Encoding': 'gzip' + (sslProtocol ? ', br' : ''), - 'Accept': '*/*' - }, - maxRedirects: options.maxRedirects || CONFIG.MAX_REDIRECTS, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - followRedirect: options.followRedirect, - jar: jar - }, options); - - var r = request(request_options) - .on('error', function(error) { - req.emit('error', error, {http2Agent: request_options.agent === http2Agent}); - }) - .on('response', function(res) { - - var contentEncoding = res.headers['content-encoding']; - contentEncoding = contentEncoding && contentEncoding.trim().toLowerCase(); - - var zlibOptions = { - flush: zlib.Z_SYNC_FLUSH, - finishFlush: zlib.Z_SYNC_FLUSH - }; - - if (contentEncoding === 'br') { - - var br = decompressStream(); - - br.request = res.request; - br.statusCode = res.statusCode; - br.headers = res.headers; - - if (!options.asBuffer) { - br.setEncoding("binary"); - } - - req.emit('response', br); - res.pipe(br); - - } else if (contentEncoding === 'gzip' || contentEncoding === 'deflate') { - - var gunzip = contentEncoding === 'gzip' ? zlib.createGunzip(zlibOptions) : zlib.createInflate(zlibOptions); - - gunzip.request = res.request; - gunzip.statusCode = res.statusCode; - gunzip.headers = res.headers; - - if (!options.asBuffer) { - gunzip.setEncoding("binary"); - } - - req.emit('response', gunzip); - res.pipe(gunzip); - - } else { - - if (!options.asBuffer) { - res.setEncoding("binary"); - } - - req.emit('response', res); - } - }); + var request_options = prepareRequestOptions({ + // TODO: jar: jar, + + // Reviewed. + uri: url, + method: 'GET', + headers: { + 'User-Agent': options.user_agent || CONFIG.USER_AGENT, + 'Accept': '*/*' + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + redirect: redirect, + follow: follow + }, options); - req.emit('request', r); + try { + fetchStreamKeepAlive(request_options) + .then(stream => { + if (!options.asBuffer) { + stream.setEncoding("binary"); + } + callbacks.onResponse && callbacks.onResponse(stream); + }) + .catch(error => { + callbacks.onError && callbacks.onError(error); + }); - } catch (ex) { - console.error('Error on getUrl for', url, '.\n Error:' + ex); - req.emit('error', ex); - } - }); - return req; + } catch (ex) { + console.error('Error on getUrl for', url, '.\n Error:' + ex); + callbacks.onError && callbacks.onError(ex); + } }; -var getHeadFunctional = exports.getHeadFunctional = function(url, options, callbacks) { - - getHead(url, options) - .on('error', function(error) { - - // Retry on ECONNRESET. - if (!options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) { - sysUtils.log(' -- getHead ' + error.code + ' first', url); - process.nextTick(function() { - getHeadFunctional(url, _.extend({}, options, {secondAttempt: true}), callbacks); - }); - return; - } else if (options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) { - sysUtils.log(' -- getHead ' + error.code + ' second', url); - } - - callbacks.onError && callbacks.onError(error); - }) - .on('request', function(request) { - callbacks.onRequest && callbacks.onRequest(request); - }) - .on('response', function(response) { - callbacks.onResponse && callbacks.onResponse(response); - }); -} - -var getHead = function(url, options) { +export const getUrlFunctional = getUrl; - var req = new events.EventEmitter(); +var getHead = function(url, options, callbacks) { var options = options || {}; + // TODO: // Store cookies between redirects and requests. - var jar = options.jar; - if (!jar) { - jar = request.jar(); + // var jar = options.jar; + // if (!jar) { + // jar = request.jar(); + // } + + var redirect, follow; + if (options.followRedirect) { + redirect = 'follow'; + follow = options.maxRedirects || CONFIG.MAX_REDIRECTS; } + if (options.followRedirect === false) { + redirect = 'manual'; + follow = 0; + } + + var request_options = prepareRequestOptions({ + // jar: jar, + + // Reviewed. + uri: url, + method: 'HEAD', + headers: { + 'User-Agent': CONFIG.USER_AGENT + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + redirect: redirect, + follow: follow + // No abort controller for head. + }); - process.nextTick(function() { - try { - var r = request(prepareRequestOptions({ - uri: url, - method: 'HEAD', - headers: { - 'User-Agent': CONFIG.USER_AGENT, - 'Connection': 'close' - }, - maxRedirects: options.maxRedirects || CONFIG.MAX_REDIRECTS, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - followRedirect: options.followRedirect, - jar: jar, - agentOptions: { - // getHead called for video, need check certificate. - rejectUnauthorized: true - } - }, { - disableHttp2: options && options.disableHttp2 - })) - .on('error', function(error) { - req.emit('error', error); - }) - .on('response', function(res) { - req.emit('response', res); - }); - - req.emit('request', r); + try { + fetchStreamAuthorized(request_options) + .then(response => { + callbacks.onResponse && callbacks.onResponse(response); + }) + .catch(error => { + callbacks.onError && callbacks.onError(error); + }); - } catch (ex) { - console.error('Error on getHead for', url, '.\n Error:' + ex); - req.emit('error', ex); - } - }); - return req; + } catch (ex) { + console.error('Error on getHead for', url, '.\n Error:' + ex); + callbacks.onError && callbacks.onError(ex); + } }; -exports.getCharset = function(string, doNotParse) { +export const getHeadFunctional = getHead; + +export function getCharset(string, doNotParse) { var charset; if (doNotParse) { @@ -356,7 +238,7 @@ exports.getCharset = function(string, doNotParse) { return charset; }; -exports.encodeText = function(charset, text) { +export function encodeText(charset, text) { try { var charset = charset || 'UTF-8'; @@ -382,7 +264,7 @@ exports.encodeText = function(charset, text) { } }; -exports.parseJSONSource = function(text, decode) { +export function parseJSONSource(text, decode) { try { return JSON.parse(decode ? entities.decodeHTML(decode(text)) : entities.decodeHTML(text)); @@ -466,7 +348,7 @@ exports.parseJSONSource = function(text, decode) { * * error == 404 if not found. * */ -exports.getImageMetadata = function(uri, options, callback){ + export function getImageMetadata(uri, options, callback){ if (typeof options === 'function') { callback = options; @@ -478,7 +360,7 @@ exports.getImageMetadata = function(uri, options, callback){ cache.withCache("image-meta:" + uri, function(callback) { var loadImageHead, imageResponseStarted, totalTime, timeout, contentLength; - var requestInstance = null; + var abortController; function finish(error, data) { @@ -490,7 +372,7 @@ exports.getImageMetadata = function(uri, options, callback){ } // We don't need more data. Abort causes error. timeout === null here so error will be skipped. - abortRequest(requestInstance) + abortController && abortController.abort(); if (!error && !data) { error = 404; @@ -560,24 +442,22 @@ exports.getImageMetadata = function(uri, options, callback){ getUrlFunctional(uri, { timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - asBuffer: true, - disableHttp2: options && options.disableHttp2 + asBuffer: true }, { - onRequest: function(req) { - requestInstance = req; - }, onResponse: function(res) { + abortController = res.abortController; + var content_type = res.headers['content-type']; if (content_type && content_type !== 'application/octet-stream' && content_type !== 'binary/octet-stream') { if (content_type.indexOf('image') === -1 && !uri.match(/\.(jpg|png|gif|webp)(\?.*)?$/i)) { - return finishCb('invalid content type: ' + res.headers['content-type'], undefined, 'onResponse content_type'); + return finishCb('invalid content type: ' + content_type, undefined, 'onResponse content_type'); } } - if (res.statusCode == 200) { + if (res.status == 200) { if (options.debug) { imageResponseStarted = totalTime(); } @@ -589,7 +469,7 @@ exports.getImageMetadata = function(uri, options, callback){ finishCb(error, data, 'imagesize'); }); } else { - finishCb(res.statusCode, undefined, 'onResponse !200'); + finishCb(res.status, undefined, 'onResponse !200'); } }, onError: function(error) { @@ -621,7 +501,7 @@ exports.getImageMetadata = function(uri, options, callback){ }, callback); }; -exports.getUriStatus = function(uri, options, callback) { +export function getUriStatus(uri, options, callback) { if (typeof options === 'function') { callback = options; @@ -664,7 +544,7 @@ exports.getUriStatus = function(uri, options, callback) { time = createTimer(); } - getUriStatus(uri, options, finish); + getUriStatusPrivate(uri, options, finish); }, { // Ignore proxy.cache_ttl, if options.cache_ttl === 0 - do not read from cache. @@ -675,11 +555,11 @@ exports.getUriStatus = function(uri, options, callback) { }, callback); }; -exports.getContentType = function(uriForCache, uriOriginal, options, cb) { +export function getContentType(uriForCache, uriOriginal, options, cb) { cache.withCache("content-type:" + uriForCache, function(cb) { - var timeout, requestInstance, totalTime; + var timeout, totalTime, abortController; function finish(error, headers) { @@ -691,7 +571,8 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) { } // We don't need more data. Abort causes error. timeout === null here so error will be skipped. - abortRequest(requestInstance); + // If 'abortController' not defined, then no request created? + abortController && abortController.abort(); var data = {}; @@ -742,14 +623,13 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) { var methodCaller = method && method === 'GET' ? getUrlFunctional : getHeadFunctional; methodCaller(uriOriginal, { - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - disableHttp2: options && options.disableHttp2 + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT }, { - onRequest: function(req) { - requestInstance = req; - }, onResponse: function(res) { - var error = res.statusCode && res.statusCode != 200 ? res.statusCode : null; + + abortController = res.abortController; + + var error = res.status && res.status != 200 ? res.status : null; if (!method // If method HEAD is not allowed. ex. Amazon S3 (=403) @@ -760,13 +640,15 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) { || error >= 500 // Or ClourFront that gobbles up headers when checking CORS. || (res.headers && !res.headers['access-control-allow-origin'] - && res.headers.server === 'AmazonS3' && !error ))) { + && res.headers['server'] === 'AmazonS3' && !error ))) { makeCall('GET'); return; } + // TODO: what is res.request.href ? + // TODO: where used res.headers.location ? if (res.request && res.request.href && res.headers && res.request.href !== uriOriginal) { - res.headers.location = res.request.href; + //res.headers.location = res.request.href; } finish(error, res.headers); @@ -787,7 +669,7 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) { }, cb); }; -exports.unifyDuration = function(duration) { +export function unifyDuration(duration) { if (duration && typeof duration === 'string') { if (duration.match(/^\d+$/)) { @@ -810,9 +692,9 @@ var NOW = new Date().getTime(); var minDate = new Date(1990, 1); -exports.unifyDate = function(date) { +export function unifyDate(date) { - if (_.isArray(date)) { + if (Array.isArray(date)) { date = date[0]; } @@ -868,7 +750,7 @@ exports.unifyDate = function(date) { }; -var lowerCaseKeys = exports.lowerCaseKeys = function(obj) { +export function lowerCaseKeys(obj) { for (var k in obj) { var lowerCaseKey = k.toLowerCase(); if (lowerCaseKey != k) { @@ -882,7 +764,7 @@ var lowerCaseKeys = exports.lowerCaseKeys = function(obj) { } }; -exports.sendLogToWhitelist = function(uri, context) { +export function sendLogToWhitelist(uri, context) { const {meta, oembed, oembedLinks, whitelistRecord} = context; @@ -939,23 +821,26 @@ exports.sendLogToWhitelist = function(uri, context) { data.oembed = oembedHref; } - request({ + // TODO: check options + fetchStream({ + qs: data, + + // Reviewed. uri: CONFIG.WHITELIST_LOG_URL, method: 'GET', - qs: data }) - .on('error', function(error) { - console.error('Error logging url:', uri, error); - }) - .on('response', function(res) { - if (res.statusCode !== 200) { - console.error('Error logging url:', uri, res.statusCode); + .then(res => { + if (res.status !== 200) { + console.error('Error logging url:', uri, res.status); } + }) + .catch(error => { + console.error('Error logging url:', uri, error); }); } }; -exports.filterLinks = function(data, options) { +export function filterLinks(data, options) { var links = data.links; @@ -1039,7 +924,7 @@ function iterateLinks(links, func) { } } -exports.generateLinksHtml = function(data, options) { +export function generateLinksHtml(data, options) { // Links may be grouped. @@ -1106,26 +991,30 @@ exports.generateLinksHtml = function(data, options) { // Private //==================================================================================== -var getUriStatus = function(uri, options, cb) { +var getUriStatusPrivate = function(uri, options, cb) { + + var request_options = prepareRequestOptions({ + // TODO: + //jar: request.jar(), //Enable cookies, uses new jar + + + // Reviewed. + uri: uri, + method: 'GET', + headers: { + 'User-Agent': CONFIG.USER_AGENT + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + follow: CONFIG.MAX_REDIRECTS + }) try { - var r = request(prepareRequestOptions({ - uri: uri, - method: 'GET', - headers: { - 'User-Agent': CONFIG.USER_AGENT - }, - maxRedirects: CONFIG.MAX_REDIRECTS, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - jar: request.jar() //Enable cookies, uses new jar - }, { - disableHttp2: options.disableHttp2 - })) - .on('error', cb) - .on('response', function(res) { - abortRequest(r); + fetchStream(request_options) + .then(res => { + // TODO: may cause AbortError before cb. + res.abortController.abort(); var data = { - code: res.statusCode, + code: res.status, content_type: res.headers && res.headers['content-type'], content_length: res.headers && res.headers['content-length'] ? parseInt(res.headers['content-length'] || '0', 10) : null }; @@ -1133,14 +1022,14 @@ var getUriStatus = function(uri, options, cb) { data.headers = res.headers; } cb(null, data); - }); - + }) + .catch(cb) } catch (ex) { cb(ex.message); } }; -var createTimer = exports.createTimer = function() { +export function createTimer() { var timer = new Date().getTime(); @@ -1260,7 +1149,7 @@ function getWhitelistLogData(meta, oembed) { return hasTrue && result; } -exports.getIframelyErrorShortCode = function(error) { +export function getIframelyErrorShortCode(error) { if (error.responseCode) { // 'http error' @@ -1275,21 +1164,12 @@ exports.getIframelyErrorShortCode = function(error) { return error.code; }; -var abortRequest = exports.abortRequest = function(request) { - if (request && request.response && request.response.h2) { - // Abort http2 with special method. - request.response.abort(); - } else if (request && !request.aborted) { - request.abort(); - } -}; - /** * @public * Generate slug for provider based on domain name of URL provided * @param url Request uri string */ -exports.getProviderName = function(url) { +export function getProviderName(url) { try { var domain = url.match(/^https?:\/\/([^\/\?#]+)/)[1]; diff --git a/lib/whitelist.js b/lib/whitelist.js index 15b356b3d..ed01ec165 100644 --- a/lib/whitelist.js +++ b/lib/whitelist.js @@ -1,13 +1,19 @@ -(function(whitelist) { + import * as chokidar from 'chokidar'; + import * as fs from 'fs'; + import * as path from 'path'; + import * as crypto from 'crypto'; + import * as _ from 'underscore'; + import * as utils from './utils.js'; + import log from '../logging.js'; + import request from 'request'; - var chokidar = require('chokidar'), - fs = require('fs'), - path = require('path'), - crypto = require('crypto'), - _ = require('underscore'), - request = require('request'), - utils = require('./utils'), - logging = require('../logging'); + import { fileURLToPath } from 'url'; + import { dirname } from 'path'; + + import CONFIG from '../config.loader.js'; + + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); var whitelistObject = {domains: {}}; var whitelistLastModified; @@ -93,7 +99,7 @@ return hash(JSON.stringify(data)); } - whitelist.findRawWhitelistRecordFor = function(uri) { + export function findRawWhitelistRecordFor(uri) { if (!whitelistObject || !whitelistObject.domains) { return null; @@ -110,7 +116,7 @@ return record; }; - whitelist.findWhitelistRecordFor = function(uri, options) { + export function findWhitelistRecordFor(uri, options) { if (!whitelistObject) { return null; @@ -165,7 +171,7 @@ return record; }; - whitelist.getWhitelistObject = function() { + export function getWhitelistObject() { return whitelistObject; }; @@ -231,7 +237,7 @@ addWildcard(); - logging.log('Whitelist activated. Domains, including blacklisted:', _.keys(data.domains).length); + log('Whitelist activated. Domains, including blacklisted:', Object.keys(data.domains).length); } function readWhitelist(filename) { @@ -333,7 +339,7 @@ if (!currentWhitelistFilename && CONFIG.WHITELIST_URL && CONFIG.WHITELIST_URL_RELOAD_PERIOD) { - logging.log("Loading whitelist from " + CONFIG.WHITELIST_URL); + log("Loading whitelist from " + CONFIG.WHITELIST_URL); var options = { uri: CONFIG.WHITELIST_URL, @@ -346,6 +352,7 @@ // Prevent caching. 'Cache-Control': 'no-cache' }, + // TODO: remove in helix-fetch gzip: true }; @@ -360,7 +367,7 @@ } else if (r.statusCode === 500) { console.error('Error loading whitelist from ' + CONFIG.WHITELIST_URL + ' : ' + newWhitelist); } else if (r.statusCode === 304) { - logging.log('Whitelist respond: 304 (not modified)'); + log('Whitelist respond: 304 (not modified)'); } else if (!newWhitelist || typeof newWhitelist === 'string') { console.error('Error loading whitelist from ' + CONFIG.WHITELIST_URL + ' : incorrect data: ' + newWhitelist); } else { @@ -373,6 +380,4 @@ setTimeout(loadWhitelistUrl, CONFIG.WHITELIST_URL_RELOAD_PERIOD); }); } - } - -})(exports); + } \ No newline at end of file diff --git a/logging.js b/logging.js index 3b0947d26..74d9e68d2 100644 --- a/logging.js +++ b/logging.js @@ -1,6 +1,6 @@ -var moment = require('moment'); +import moment from 'moment'; -exports.log = function() { +export default function log() { var args = Array.prototype.slice.apply(arguments); // Add ip if request provided. diff --git a/modules/api/utils.js b/modules/api/utils.js index e0993d69b..a2f2d23c2 100644 --- a/modules/api/utils.js +++ b/modules/api/utils.js @@ -1,6 +1,6 @@ var _RE = /^_.+/; -exports.getProviderOptionsQuery = function(query) { +export function getProviderOptionsQuery(query) { var providerOptionsQuery = {}; for(var key in query) { @@ -28,7 +28,7 @@ function normalizeValue(value) { return value; } -exports.getProviderOptionsFromQuery = function(query) { +export function getProviderOptionsFromQuery(query) { /* Convert '_option=value' to providerOptions = { diff --git a/modules/api/views.js b/modules/api/views.js index 181296716..76ca0ac08 100644 --- a/modules/api/views.js +++ b/modules/api/views.js @@ -1,15 +1,15 @@ -var iframelyCore = require('../../lib/core'); -var utils = require('../../utils'); -var _ = require('underscore'); -var async = require('async'); -var cache = require('../../lib/cache'); -var iframelyUtils = require('../../lib/utils'); -var oembedUtils = require('../../lib/oembed'); -var whitelist = require('../../lib/whitelist'); -var pluginLoader = require('../../lib/loader/pluginLoader'); -var jsonxml = require('jsontoxml'); -var url = require('url'); -var apiUtils = require('./utils'); +import * as iframelyCore from '../../lib/core.js'; +import * as utils from '../../utils.js'; +import * as _ from 'underscore'; +import * as async from 'async'; +import { cache } from '../../lib/cache.js'; +import * as iframelyUtils from '../../lib/utils.js'; +import * as oembedUtils from '../../lib/oembed.js'; +import * as whitelist from '../../lib/whitelist.js'; +import * as pluginLoader from '../../lib/loader/pluginLoader.js'; +import * as jsonxml from 'jsontoxml'; +import * as url from 'url'; +import * as apiUtils from './utils.js'; var getProviderOptionsQuery = apiUtils.getProviderOptionsQuery; var getProviderOptionsFromQuery = apiUtils.getProviderOptionsFromQuery; @@ -35,7 +35,9 @@ function prepareUri(uri) { var log = utils.log; -var version = require('../../package.json').version; +import { readFile } from 'fs/promises'; +const json = JSON.parse(await readFile(new URL('../../package.json', import.meta.url))); +var version = json.version; function getRenderLinkCacheKey(uri, req) { var query = getProviderOptionsQuery(req.query); @@ -106,7 +108,7 @@ function processInitialErrors(uri, next) { } } -module.exports = function(app) { +export default function(app) { app.get('/iframely', function(req, res, next) { diff --git a/modules/debug/views.js b/modules/debug/views.js index 5b9314aa6..9aae734da 100644 --- a/modules/debug/views.js +++ b/modules/debug/views.js @@ -1,6 +1,6 @@ -var getProviderOptionsQuery = require('../api/utils').getProviderOptionsQuery; +import { getProviderOptionsQuery } from '../api/utils.js'; -module.exports = function(app) { +export default function(app) { app.get('/debug', function(req, res, next) { diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js index 890e2cf32..46f00f783 100644 --- a/modules/tests-ui/models.js +++ b/modules/tests-ui/models.js @@ -1,28 +1,15 @@ -(function() { - - if (!CONFIG.tests) { - return; + import moment from 'moment'; + import mongoose from 'mongoose'; + import CONFIG from '../../config.loader.js'; + + mongoose.set('useUnifiedTopology', true); + mongoose.set('useCreateIndex', true); + mongoose.set('useNewUrlParser', true); + if (global.Promise) { + mongoose.Promise = global.Promise; } - var moment = require('moment'); - - var mongoose, db; - - // DB connect. - try { - mongoose = require('mongoose'); - mongoose.set('useUnifiedTopology', true); - mongoose.set('useCreateIndex', true); - mongoose.set('useNewUrlParser', true); - if (global.Promise) { - mongoose.Promise = global.Promise; - } - db = mongoose.createConnection(CONFIG.tests.mongodb); - } catch (ex) { - console.error("Plugins testing framework will not work. Can't connect to mongodb."); - console.error(ex.stack); - return; - } + const db = mongoose.createConnection(CONFIG.tests.mongodb); var Schema = mongoose.Schema; @@ -169,9 +156,7 @@ return moment(this.created_at).format("DD-MM-YY HH:mm"); }; - exports.PluginTest = db.model('PluginTest', PluginTestSchema); - exports.PageTestLog = db.model('PageTestLog', PageTestLogSchema); - exports.TestUrlsSet = db.model('TestUrlsSet', TestUrlsSetSchema); - exports.TestingProgress = db.model('TestingProgress', TestingProgressSchema); - -})(); \ No newline at end of file + export const PluginTest = db.model('PluginTest', PluginTestSchema); + export const PageTestLog = db.model('PageTestLog', PageTestLogSchema); + export const TestUrlsSet = db.model('TestUrlsSet', TestUrlsSetSchema); + export const TestingProgress = db.model('TestingProgress', TestingProgressSchema); \ No newline at end of file diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js index 8fb7a9fe6..01c7f2048 100644 --- a/modules/tests-ui/tester.js +++ b/modules/tests-ui/tester.js @@ -1,23 +1,21 @@ -global.CONFIG = require('../../config'); +import CONFIG from '../../config.loader.js'; +global.CONFIG = CONFIG; if (!CONFIG.tests) { console.error('Tests not started: CONFIG.tests not configured.'); process.exit(0); - return; + // return; } process.title = "iframely-tests"; -process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; -var async = require('async'); -var _ = require('underscore'); - -var models = require('./models'); -var utils = require('./utils'); - -var iframely = require('../../lib/core').run; -var whitelist = require('../../lib/whitelist'); -var pluginLoader = require('../../lib/loader/pluginLoader'); +import * as async from 'async'; +import * as _ from 'underscore'; +import * as models from './models.js'; +import * as utils from './utils.js'; +import { run as iframely } from '../../lib/core.js'; +import * as whitelist from '../../lib/whitelist.js'; +import * as pluginLoader from '../../lib/loader/pluginLoader.js'; var plugins = pluginLoader._plugins; var testOnePlugin = false; @@ -49,11 +47,6 @@ var PageTestLog = models.PageTestLog; var TestUrlsSet = models.TestUrlsSet; var TestingProgress = models.TestingProgress; -if (!PluginTest) { - process.exit(0); - return; -} - function log() { if (CONFIG.DEBUG) { console.log.apply(console, arguments); diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js index 388040982..6d0f58308 100644 --- a/modules/tests-ui/utils.js +++ b/modules/tests-ui/utils.js @@ -1,20 +1,14 @@ -var _ = require('underscore'); -var FeedParser = require('feedparser'); -var request = require('request'); -var async = require('async'); -var url = require('url'); +import * as _ from 'underscore'; +import FeedParser from 'feedparser'; +import request from 'request'; +import * as async from 'async'; +import * as url from 'url'; +import { PageTestLog, TestUrlsSet, PluginTest } from './models.js'; +import { findWhitelistRecordFor } from '../../lib/whitelist.js'; +import { getPluginData as iframelyGetPluginData } from '../../lib/core.js'; +import * as pluginLoader from '../../lib/loader/pluginLoader.js'; +import * as pluginUtils from '../../lib/loader/utils.js'; -var models = require('./models'); -var PageTestLog = models.PageTestLog; -var TestUrlsSet = models.TestUrlsSet; -var PluginTest = models.PluginTest; - -var findWhitelistRecordFor = require('../../lib/whitelist').findWhitelistRecordFor; - -var iframelyGetPluginData = require('../../lib/core').getPluginData; - -var pluginLoader = require('../../lib/loader/pluginLoader'); -var pluginUtils = require('../../lib/loader/utils'); var plugins = pluginLoader._plugins, pluginsList = pluginLoader._pluginsList, DEFAULT_PARAMS = [].concat(pluginUtils.DEFAULT_PARAMS, pluginUtils.POST_PLUGIN_DEFAULT_PARAMS), @@ -37,7 +31,7 @@ const COLORS = { const SLACK_USERNAME = "Testy"; -exports.sendQANotification = function(logEntry, data) { +export function sendQANotification(logEntry, data) { if (CONFIG.SLACK_WEBHOOK_FOR_QA && CONFIG.SLACK_CHANNEL_FOR_QA) { @@ -93,7 +87,7 @@ exports.sendQANotification = function(logEntry, data) { } function getTestsSummary(cb) { - exports.loadPluginTests(function(error, pluginTests) { + loadPluginTests(function(error, pluginTests) { pluginTests.forEach(function(pluginTest) { @@ -129,7 +123,7 @@ function getTestsSummary(cb) { }); } -exports.loadPluginTests = function(cb) { +export function loadPluginTests(cb) { var pluginTests; @@ -216,7 +210,7 @@ exports.loadPluginTests = function(cb) { }); } -exports.getPluginUnusedMethods = function(pluginId, debugData) { +export function getPluginUnusedMethods(pluginId, debugData) { var usedMethods = getAllUsedMethods(debugData); var pluginMethods = findAllPluginMethods(pluginId, plugins); @@ -228,7 +222,7 @@ exports.getPluginUnusedMethods = function(pluginId, debugData) { }; }; -exports.getErrors = function(debugData) { +export function getErrors(debugData) { var errors = []; @@ -248,7 +242,7 @@ exports.getErrors = function(debugData) { var MAX_FEED_URLS = 5; -var fetchFeedUrls = exports.fetchFeedUrls = function(feedUrl, options, cb) { +export function fetchFeedUrls(feedUrl, options, cb) { if (typeof options === "function") { cb = options; @@ -266,7 +260,12 @@ var fetchFeedUrls = exports.fetchFeedUrls = function(feedUrl, options, cb) { cb(error, urls); }; - request(feedUrl) + request({ + uri: feedUrl, + agentOptions: { + rejectUnauthorized: false + } + }) .pipe(new FeedParser({addmeta: false})) .on('error', function(error) { _cb(error); @@ -300,7 +299,7 @@ var fetchFeedUrls = exports.fetchFeedUrls = function(feedUrl, options, cb) { }); }; -exports.fetchUrlsByPageOnFeed = function(pageWithFeed, otpions, cb) { +export function fetchUrlsByPageOnFeed(pageWithFeed, otpions, cb) { if (typeof options === "function") { cb = options; @@ -343,7 +342,7 @@ exports.fetchUrlsByPageOnFeed = function(pageWithFeed, otpions, cb) { ], cb); }; -exports.fetchUrlsByPageAndSelector = function(page, selector, options, cb) { +export function fetchUrlsByPageAndSelector(page, selector, options, cb) { if (typeof options === "function") { cb = options; @@ -356,14 +355,14 @@ exports.fetchUrlsByPageAndSelector = function(page, selector, options, cb) { iframelyGetPluginData(page, 'cheerio', findWhitelistRecordFor, cb); }, - function($, cb) { + function(cheerio, cb) { - var $links = $(selector); + var $links = cheerio(selector); var urls = []; $links.each(function() { if (urls.length < MAX_FEED_URLS) { - var href = $(this).attr(options.urlAttribute || "href"); + var href = cheerio(this).attr(options.urlAttribute || "href"); if (href) { var href = url.resolve(page, href); if (urls.indexOf(href) == -1) { diff --git a/modules/tests-ui/views.js b/modules/tests-ui/views.js index 8f4702826..ba411589e 100644 --- a/modules/tests-ui/views.js +++ b/modules/tests-ui/views.js @@ -1,22 +1,14 @@ -(function() { - - if (!CONFIG.tests) { - module.exports = function(){}; - return; - } - - var async = require('async'); - var moment = require('moment'); - var _ = require('underscore'); - var exec = require('child_process').exec; - - var models = require('./models'); - var utils = require('./utils'); + import * as async from 'async'; + import moment from 'moment'; + import * as _ from 'underscore'; + import { exec as exec } from 'child_process'; + import * as models from './models.js'; + import * as utils from './utils.js'; var PluginTest = models.PluginTest; var TestingProgress = models.TestingProgress; - module.exports = function(app){ + export default function(app){ app.get('/tests/run/:plugin', function(req, res, next) { @@ -158,5 +150,4 @@ }); }); }); - } -})(); + } \ No newline at end of file diff --git a/package.json b/package.json index 11cef840c..239fa5d46 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "iframely", "version": "1.6.1", + "type": "module", "description": "oEmbed/2 gateway endpoint. Get embed data for various http links through one self-hosted API", "keywords": [ "oembed", @@ -19,17 +20,17 @@ }, "license": "MIT", "dependencies": { + "@adobe/helix-fetch": "^3.0.0", "async": "2.4.1", - "cheerio": "0.22.0", + "cheerio": "^0.22.0", "chokidar": "^3.3.1", "ejs": "^3.1.6", "entities": "1.1.1", "express": "^4.16.3", + "globby": "^12.0.2", "graceful-cluster": "0.0.3", - "htmlparser2": "3.9.2", - "http-parser-js": "itteco/http-parser-js#magicode-fix-22", - "iconv-lite": "0.4.17", - "iltorb": "^2.4.3", + "htmlparser2": "^7.1.2", + "iconv-lite": "^0.6.3", "jslint": "^0.12.1", "jsontoxml": "0.0.11", "memcached": "2.2.2", @@ -55,7 +56,7 @@ "supertest": "^4.0.2" }, "iframely-proxy-plugins": true, - "main": "./lib/core", + "main": "./lib/core.js", "scripts": { "test": "npm run test-core-plugins && npm run test-e2e", "test-core-plugins": "mocha --exit test/core-plugins.js", diff --git a/plugins/custom/domain-icon.js b/plugins/custom/domain-icon.js index 1b4b332a7..f99571706 100644 --- a/plugins/custom/domain-icon.js +++ b/plugins/custom/domain-icon.js @@ -1,13 +1,10 @@ // use this mixin for domain plugins where you do not want to pull out htmlparser but do need an icon or logo +import { cache } from '../../lib/cache.js'; +import * as async from 'async'; +import * as _ from 'underscore'; +import log from '../../logging.js'; -var core = require('../../lib/core'); -var cache = require('../../lib/cache'); -var async = require('async'); -var _ = require('underscore'); - -var log = exports.log = require('../../logging').log; - -module.exports = { +export default { provides: 'domain_icons', @@ -15,7 +12,7 @@ module.exports = { return domain_icons; }, - getData: function(url, cb, options) { + getData: function(url, iframelyRun, options, cb) { // find domain and protocol var domain, protocol; @@ -73,7 +70,7 @@ module.exports = { // + run icons validation right away // forceSyncCheck - ask 'checkFavicon' to check favicon this time before callback. - core.run(domainUri, _.extend({}, options, {forceSyncCheck: true}), function(error, data) { + iframelyRun(domainUri, _.extend({}, options, {forceSyncCheck: true}), function(error, data) { var icons; diff --git a/plugins/custom/fb-error.js b/plugins/custom/fb-error.js index ea429000e..89620c5ff 100644 --- a/plugins/custom/fb-error.js +++ b/plugins/custom/fb-error.js @@ -1,6 +1,6 @@ -var logging = require('../../logging'); +import log from '../../logging.js'; -module.exports = { +export default { getLink: function(oembedError, url, cb) { @@ -49,7 +49,7 @@ module.exports = { result.message = fbError.message; } - logging.log('Facebook oembed api - error getting oembed for', url, JSON.stringify(fbError), JSON.stringify(result)); + log('Facebook oembed api - error getting oembed for', url, JSON.stringify(fbError), JSON.stringify(result)); return cb(result); }, diff --git a/plugins/custom/http-headers.js b/plugins/custom/http-headers.js index c3de6bbf5..2e8a7732c 100644 --- a/plugins/custom/http-headers.js +++ b/plugins/custom/http-headers.js @@ -1,12 +1,12 @@ -module.exports = { +export default { provides: "headers", getData: function(htmlparser, cb) { - if (htmlparser.request && htmlparser.request.response && htmlparser.request.response.headers) { + if (htmlparser.headers) { return cb (null, { - headers: htmlparser.request.response.headers + headers: htmlparser.headers }) } else { cb(); diff --git a/plugins/custom/noindex/noindex-header.js b/plugins/custom/noindex/noindex-header.js index 4c91f1c83..19944f13c 100644 --- a/plugins/custom/noindex/noindex-header.js +++ b/plugins/custom/noindex/noindex-header.js @@ -1,9 +1,9 @@ -var pluginUtils = require('./utils'); +import * as pluginUtils from './utils.js'; -module.exports = { +export default { getData: function(htmlparser, cb) { - var headers = htmlparser.request.response.headers; + var headers = htmlparser.headers; if (pluginUtils.checkRobots(headers['x-robots-tag'], cb)) { return; } else { diff --git a/plugins/custom/noindex/noindex-meta.js b/plugins/custom/noindex/noindex-meta.js index ef08942d2..83ffbd843 100644 --- a/plugins/custom/noindex/noindex-meta.js +++ b/plugins/custom/noindex/noindex-meta.js @@ -1,6 +1,6 @@ -var pluginUtils = require('./utils'); +import * as pluginUtils from './utils.js'; -module.exports = { +export default { getData: function(meta, cb) { if (pluginUtils.checkRobots(meta.robots, cb)) { diff --git a/plugins/custom/noindex/utils.js b/plugins/custom/noindex/utils.js index 4f8e0889c..2b653581d 100644 --- a/plugins/custom/noindex/utils.js +++ b/plugins/custom/noindex/utils.js @@ -1,8 +1,8 @@ -exports.notPlugin = true; +export const notPlugin = true; var NO_INDEX_TAGS = ['noindex']; -exports.checkRobots = function(noindexHeader, cb) { +export function checkRobots(noindexHeader, cb) { if (noindexHeader) { var i; for(i = 0; i < NO_INDEX_TAGS.length; i++) { diff --git a/plugins/custom/oembed-error.js b/plugins/custom/oembed-error.js index a320c5063..b0a88bffd 100644 --- a/plugins/custom/oembed-error.js +++ b/plugins/custom/oembed-error.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getData: function(oembedError, cb) { diff --git a/plugins/custom/og-image-rel-image.js b/plugins/custom/og-image-rel-image.js index 370877d03..ede29456c 100644 --- a/plugins/custom/og-image-rel-image.js +++ b/plugins/custom/og-image-rel-image.js @@ -1,4 +1,4 @@ -var _ = require("underscore"); +import * as _ from "underscore"; var rel = [CONFIG.R.image, CONFIG.R.og]; @@ -18,7 +18,7 @@ function getImageLinks(image) { }]; } -module.exports = { +export default { getLinks: function(og) { diff --git a/plugins/custom/query.js b/plugins/custom/query.js new file mode 100644 index 000000000..ace40f385 --- /dev/null +++ b/plugins/custom/query.js @@ -0,0 +1,17 @@ +import * as URL from "url" + +export default { + + provides: 'query', + + getData: function(url) { + + if (/\?/i.test(url)) { + return { + query: URL.parse(url, true).query + }; + } else { + return {query: {}} + } + } +} \ No newline at end of file diff --git a/plugins/custom/twitter-image-rel-image.js b/plugins/custom/twitter-image-rel-image.js index 828df579f..d6c122372 100644 --- a/plugins/custom/twitter-image-rel-image.js +++ b/plugins/custom/twitter-image-rel-image.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(twitter) { diff --git a/plugins/domains/500px/500px.com.js b/plugins/domains/500px/500px.com.js index bc2a01a76..d0a05aeb5 100644 --- a/plugins/domains/500px/500px.com.js +++ b/plugins/domains/500px/500px.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/(?:web\.)?500px\.com\/photo\/(\d+)/i, diff --git a/plugins/domains/absnews.go.com.js b/plugins/domains/absnews.go.com.js index c575b940c..6a876ff6e 100644 --- a/plugins/domains/absnews.go.com.js +++ b/plugins/domains/absnews.go.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/abcnews\.go\.com\/\w+\/(?:\w+\/)?video\/[a-zA-Z0-9\-_]+\-(\d+)/i diff --git a/plugins/domains/animoto.com.js b/plugins/domains/animoto.com.js index bf983ce7b..0bc060041 100644 --- a/plugins/domains/animoto.com.js +++ b/plugins/domains/animoto.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/animoto\.com\/play\/\w+/i, diff --git a/plugins/domains/archive.org.js b/plugins/domains/archive.org.js index ef7dd4e38..2a19def8c 100644 --- a/plugins/domains/archive.org.js +++ b/plugins/domains/archive.org.js @@ -1,6 +1,6 @@ -var utils = require('../../lib/utils'); +import * as utils from '../../lib/utils.js'; -module.exports = { +export default { re: [ /^https?:\/\/archive\.org\/details\/([^\/]+)\/?\??/i diff --git a/plugins/domains/art19.com.js b/plugins/domains/art19.com.js index 96806681b..fdf55b2dd 100644 --- a/plugins/domains/art19.com.js +++ b/plugins/domains/art19.com.js @@ -1,8 +1,4 @@ -var $ = require('cheerio'); -const querystring = require('querystring'); -const URL = require("url"); - -module.exports = { +export default { re: [ /^(https?:\/\/art19\.com\/shows\/[a-zA-Z0-9\-_]+\/episodes\/[a-zA-Z0-9\-_]+)/i, @@ -14,68 +10,55 @@ module.exports = { "oembed-description", "og-image", "oembed-site", - "domain-icon" + "domain-icon", + "oembed-iframe" ], - getLink: function(oembed, options) { - - if (oembed.html) { - - var $container = $('<div>'); - try { - $container.html(oembed.html); - } catch(ex) {} - - var $iframe = $container.find('iframe'); - - if ($iframe.length == 1) { + getLink: function(iframe, options) { - var player = $iframe.attr('src'); - var params = URL.parse(player, true).query; + var params = Object.assign(iframe.query); - var theme = options.getRequestOptions('players.theme', 'light'); - params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue'; + var theme = options.getRequestOptions('players.theme', 'light'); + params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue'; - var opts = {}; + var opts = {}; - var horizontal = options.getRequestOptions('players.horizontal', true); + var horizontal = options.getRequestOptions('players.horizontal', true); - if (horizontal) { - delete params.type; - delete params.stretch; + if (horizontal) { + delete params.type; + delete params.stretch; - var theme = options.getRequestOptions('players.theme', 'light'); - params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue'; + var theme = options.getRequestOptions('players.theme', 'light'); + params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue'; - opts.theme = { - label: CONFIG.L.theme, - value: theme, - values: { - light: CONFIG.L.light, - dark: CONFIG.L.dark - } - }; - } else { - params.type = 'artwork'; - params.stretch = true; - delete params.theme; - } - - opts.horizontal = { - label: CONFIG.L.horizontal, - value: horizontal + opts.theme = { + label: CONFIG.L.theme, + value: theme, + values: { + light: CONFIG.L.light, + dark: CONFIG.L.dark } + }; + } else { + params.type = 'artwork'; + params.stretch = true; + delete params.theme; + } - return { - href: (/\?/.test(player) ? player.replace(/\?.+/, '?') : player + '?') + querystring.stringify(params), - type: CONFIG.T.text_html, - rel: [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.oembed], // keep rel oembed here - it prevents validators from removing embed srcz - media: horizontal ? {height: oembed.height, scrolling: 'no'} : {'aspect-ratio': 1}, - scrolling: 'no', - options: opts - }; - } + opts.horizontal = { + label: CONFIG.L.horizontal, + value: horizontal } + + return { + href: iframe.assignQuerystring(params), + type: CONFIG.T.text_html, + rel: [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.oembed], // keep rel oembed here - it prevents validators from removing embed srcz + media: horizontal ? {height: iframe.height, scrolling: 'no'} : {'aspect-ratio': 1}, + scrolling: 'no', + options: opts + }; }, tests: [{ diff --git a/plugins/domains/bandcamp.com.js b/plugins/domains/bandcamp.com.js index 12bc2108d..547908ff4 100644 --- a/plugins/domains/bandcamp.com.js +++ b/plugins/domains/bandcamp.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/[a-z0-9-]+\.bandcamp\.com\/(album|track)\/(\w+)/i, diff --git a/plugins/domains/beatport.com.js b/plugins/domains/beatport.com.js index e77981099..f57971399 100644 --- a/plugins/domains/beatport.com.js +++ b/plugins/domains/beatport.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(?:www|pro)\.beatport\.com\/(track|mix)\/[a-zA-Z0-9\.\-]+\/(\d+)/i diff --git a/plugins/domains/bigthink.com.js b/plugins/domains/bigthink.com.js index d599e035e..702771309 100644 --- a/plugins/domains/bigthink.com.js +++ b/plugins/domains/bigthink.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { mixins: [ "*" diff --git a/plugins/domains/box.com.js b/plugins/domains/box.com.js index 66c1889c0..e1bcbbbf0 100644 --- a/plugins/domains/box.com.js +++ b/plugins/domains/box.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https:\/\/app\.box\.com\/(?:embed|embed_widget)?\/?s\/([a-zA-Z0-9]+)/, diff --git a/plugins/domains/brainyquote.com.js b/plugins/domains/brainyquote.com.js index 2e93c9a94..09491f91c 100644 --- a/plugins/domains/brainyquote.com.js +++ b/plugins/domains/brainyquote.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.)?brainyquote\.com\/quotes(?:\/quotes)?\/\w\/?/i diff --git a/plugins/domains/brightcove.com/players.brightcove.net.js b/plugins/domains/brightcove.com/players.brightcove.net.js index 6ac245852..c0e1d164f 100644 --- a/plugins/domains/brightcove.com/players.brightcove.net.js +++ b/plugins/domains/brightcove.com/players.brightcove.net.js @@ -1,7 +1,6 @@ -var cheerio = require('cheerio'); -var utils = require('../../../lib/utils'); +import * as utils from '../../../lib/utils.js'; -module.exports = { +export default { re: [ /^https?:\/\/players\.brightcove\.net\/\d+\/[a-zA-Z0-9]+_[a-zA-Z0-9]+\/index\.html\?videoId=\d+/i @@ -12,15 +11,16 @@ module.exports = { mixins: [ "oembed-title", "oembed-site", - "oembed-error" + "oembed-error", + "oembed-iframe" ], //HTML parser will 404 if BC account or player does not exist. - getLinks: function(url, oembed, options, cb) { + getLinks: function(url, iframe, options, cb) { var player = { type: CONFIG.T.text_html, - rel: [CONFIG.R.oembed, CONFIG.R.player, CONFIG.R.html5] + rel: [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.oembed] }; // autoplay=true comes from `brightcove-in-page-promo` only and follows whitelistRecord @@ -30,15 +30,8 @@ module.exports = { player.autoplay = "autoplay=true"; } - var $container = cheerio('<div>'); - try { - $container.html(oembed.html); - } catch (ex) {} - - var $iframe = $container.find('iframe'); - - if ($iframe.length == 1) { - player.href = $iframe.attr('src') + (/&autoplay=true/.test(url) ? '&autoplay=true' : ''); // autoplay=true in URL comes from brightcove-allow-in-page whitelist record + if (iframe.src) { + player.href = iframe.src + (/&autoplay=true/.test(url) ? '&autoplay=true' : ''); // autoplay=true in URL comes from brightcove-allow-in-page whitelist record } if (/&iframe-url=/.test(url)) { @@ -49,9 +42,9 @@ module.exports = { player.accept = CONFIG.T.text_html; // verify that it exists and isn't X-Frame-Optioned } - if (oembed.thumbnail_url) { + if (iframe.placeholder) { - utils.getImageMetadata(oembed.thumbnail_url, options, function(error, data) { + utils.getImageMetadata(iframe.placeholder, options, function(error, data) { var links = []; @@ -62,7 +55,7 @@ module.exports = { } else if (data.width && data.height) { links.push({ - href: oembed.thumbnail_url, + href: iframe.placeholder, type: CONFIG.T.image, rel: CONFIG.R.thumbnail, width: data.width, @@ -70,14 +63,14 @@ module.exports = { }); } - player['aspect-ratio'] = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height; + player['aspect-ratio'] = (data.width && data.height) ? data.width / data.height : iframe.width / iframe.height; links.push(player); - cb(null, links); + return cb(null, links); }); } else { - cb (null, player); + return cb (null, player); } }, diff --git a/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js b/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js index 1e5367a38..150b36d37 100644 --- a/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js +++ b/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.buzzfeed\.com\//i diff --git a/plugins/domains/buzzfeed.com/buzzfeed.video.js b/plugins/domains/buzzfeed.com/buzzfeed.video.js index 78369ab49..a4a3e65f0 100644 --- a/plugins/domains/buzzfeed.com/buzzfeed.video.js +++ b/plugins/domains/buzzfeed.com/buzzfeed.video.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: '__promoUri', diff --git a/plugins/domains/c-span.org/c-span.org-options.js b/plugins/domains/c-span.org/c-span.org-options.js index 2eea886d6..13535429f 100644 --- a/plugins/domains/c-span.org/c-span.org-options.js +++ b/plugins/domains/c-span.org/c-span.org-options.js @@ -1,6 +1,8 @@ -module.exports = { +import c_span_org from './c-span.org.js'; - re: require('./c-span.org.js').re, +export default { + + re: c_span_org.re, getData: function(url, options) { options.exposeStatusCode = true; diff --git a/plugins/domains/c-span.org/c-span.org.js b/plugins/domains/c-span.org/c-span.org.js index acd11cc54..147f98426 100644 --- a/plugins/domains/c-span.org/c-span.org.js +++ b/plugins/domains/c-span.org/c-span.org.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.c-span\.org\/video\/\?(c?[\d-]+)(\/[\w-]+)/i, diff --git a/plugins/domains/cartodb.com.js b/plugins/domains/cartodb.com.js index f10d2b3cf..8ebbf26ab 100644 --- a/plugins/domains/cartodb.com.js +++ b/plugins/domains/cartodb.com.js @@ -1,6 +1,4 @@ -var cheerio = require('cheerio'); - -module.exports = { +export default { re: /^https?:(\/\/[\w-]+\.carto(?:db)?\.com\/(?:u\/[\w-]+\/)?viz\/[a-z0-9-]+)/i, @@ -13,7 +11,8 @@ module.exports = { "oembed-author", "oembed-site", "keywords", - "favicon" + "favicon", + "oembed-iframe" ], getMeta: function(url, meta) { @@ -23,27 +22,15 @@ module.exports = { }; }, - getLink: function(oembed) { - - var $container = cheerio('<div>'); - try { - $container.html(oembed.html5 || oembed.html); - } catch (ex) {} - - var $iframe = $container.find('iframe'); - - if ($iframe.length == 1) { - - return { - href: $iframe.attr('src'), - type: CONFIG.T.text_html, - rel: [CONFIG.R.app, CONFIG.R.ssl, CONFIG.R.html5], - "aspect-ratio": 4/3, - "padding-bottom": 30 - // aspect 4:3 is better than height=520px and width=100% - }; + getLink: function(iframe) { + return { + href: iframe.src, + type: CONFIG.T.text_html, + rel: [CONFIG.R.app, CONFIG.R.ssl, CONFIG.R.html5], + "aspect-ratio": 4/3, + "padding-bottom": 30 + // aspect 4:3 is better than height=520px and width=100% } - }, tests: [{ diff --git a/plugins/domains/cbc.ca.js b/plugins/domains/cbc.ca.js index 956bad3a6..5dc0a5b40 100644 --- a/plugins/domains/cbc.ca.js +++ b/plugins/domains/cbc.ca.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.cbc\.ca\/player\//i, diff --git a/plugins/domains/channel9.msdn.com.js b/plugins/domains/channel9.msdn.com.js index b94b4b422..38b57e183 100644 --- a/plugins/domains/channel9.msdn.com.js +++ b/plugins/domains/channel9.msdn.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/channel9\.msdn\.com\/Events\/([\w-]+\/\d+\/[\w-]+)/i, diff --git a/plugins/domains/cnevids.com.js b/plugins/domains/cnevids.com.js index d0aa03ab0..105d6c72e 100644 --- a/plugins/domains/cnevids.com.js +++ b/plugins/domains/cnevids.com.js @@ -1,22 +1,19 @@ -module.exports = { +export default { mixins: [ "oembed-thumbnail", "oembed-author", "oembed-site", - "oembed-title" + "oembed-title", + "oembed-iframe" ], - getLink: function(oembed, whitelistRecord) { + getLink: function(iframe, oembed, whitelistRecord) { - var iframe = oembed.getIframe(); - - if (iframe && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.video', 'autoplay')) { - - var href = iframe.src; + if (iframe.src && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.video', 'autoplay')) { var links = [{ - href: href + (href.indexOf('?') > -1 ? '&' : '?') + 'autoplay=0', + href: iframe.replaceQuerystring({autoplay:0}), type: CONFIG.T.text_html, rel: [CONFIG.R.player, CONFIG.R.oembed, CONFIG.R.html5], autoplay: "autoplay=1", diff --git a/plugins/domains/cnn.com.js b/plugins/domains/cnn.com.js index 40f2e1f04..7345613d9 100644 --- a/plugins/domains/cnn.com.js +++ b/plugins/domains/cnn.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(www|edition)?\.?cnn\.com\/videos?\//i, diff --git a/plugins/domains/codepen.io.js b/plugins/domains/codepen.io.js index 7fe20444a..05dc128a3 100644 --- a/plugins/domains/codepen.io.js +++ b/plugins/domains/codepen.io.js @@ -1,8 +1,4 @@ -const $ = require('cheerio'); -const querystring = require('querystring'); -const URL = require("url"); - -module.exports = { +export default { re: /https?:\/\/codepen\.io\/(?:[a-z0-9\-_]+\/)?(pen|details|full)\/([a-z0-9\-]+)/i, @@ -11,11 +7,12 @@ module.exports = { "oembed-author", "oembed-site", "oembed-title", + "oembed-iframe", //"description", // don't enable to avoid 403 from CodePen's htmlparser. Description is '...' in most cases anyway "domain-icon" ], - getLink: function(oembed, options) { + getLink: function(oembed, iframe, options) { if (oembed.author_url === "https://codepen.io/anon/") { return { // And no fallback to generics @@ -23,57 +20,47 @@ module.exports = { } } - var $container = $('<div>'); - try{ - $container.html(oembed.html); - } catch(ex) {} - - var $iframe = $container.find('iframe'); - - if ($iframe.length == 1) { - - var href = $iframe.attr('src'); - var params = URL.parse(href, true).query; + var params = Object.assign(iframe.query); - var click_to_load = options.getRequestOptions('codepen.click_to_load', /\/embed\/preview\//.test(href)); - href = href.replace(/\/embed\/(?:preview\/)?/, '/embed/').replace(/\/embed\//, '/embed/' + (click_to_load ? 'preview/' : '')); + params.height = options.getRequestOptions('codepen.height', oembed.height); - params.height = options.getRequestOptions('codepen.height', oembed.height); + var theme = options.getRequestOptions('players.theme', params.theme || 'auto'); - var theme = options.getRequestOptions('players.theme', params.theme || 'auto'); - - if (theme === 'auto') { - delete params['theme-id']; - } else { - params['theme-id'] = theme; - } + if (theme === 'auto') { + delete params['theme-id']; + } else { + params['theme-id'] = theme; + } - return { - href: href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1'), - type: CONFIG.T.text_html, - rel: [CONFIG.R.app, CONFIG.R.oembed, CONFIG.R.html5], - height: params.height, - options: { - height: { - label: CONFIG.L.height, - value: params.height, - placeholder: 'ex.: 600, in px' - }, - click_to_load: { - label: 'Use click-to-load', - value: click_to_load - }, - theme: { - label: CONFIG.L.theme, - value: theme, - values: { - light: CONFIG.L.light, - dark: CONFIG.L.dark, - auto: CONFIG.L.default - } + var href = iframe.assignQuerystring(params); + var click_to_load = options.getRequestOptions('codepen.click_to_load', /\/embed\/preview\//.test(href)); + href = href.replace(/\/embed\/(?:preview\/)?/, '/embed/').replace(/\/embed\//, '/embed/' + (click_to_load ? 'preview/' : '')); + + return { + href: href, + type: CONFIG.T.text_html, + rel: [CONFIG.R.app, CONFIG.R.oembed, CONFIG.R.html5], + height: params.height, + options: { + height: { + label: CONFIG.L.height, + value: params.height, + placeholder: 'ex.: 600, in px' + }, + click_to_load: { + label: 'Use click-to-load', + value: click_to_load + }, + theme: { + label: CONFIG.L.theme, + value: theme, + values: { + light: CONFIG.L.light, + dark: CONFIG.L.dark, + auto: CONFIG.L.default } } - }; + } } }, diff --git a/plugins/domains/d.pr.js b/plugins/domains/d.pr.js index 3e8542754..852b43da8 100644 --- a/plugins/domains/d.pr.js +++ b/plugins/domains/d.pr.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(\w+\.)?d\.pr(?:\/free)?\/(?:i|v|)\//i diff --git a/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js b/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js index 36d07521e..cf9203b6d 100644 --- a/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js +++ b/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js @@ -1,4 +1,4 @@ -module.exports = { +export default { // direct "share" links to players from DailyMail articles. They end with #v-1467332342001 re: [ diff --git a/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js b/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js index a8cc52a61..8c5fdfb66 100644 --- a/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js +++ b/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.dailymail\.co\.uk\/video\/\w+\/video\-(\d+)\//i diff --git a/plugins/domains/dailymail.co.uk/dailymail.video.js b/plugins/domains/dailymail.co.uk/dailymail.video.js index dbca24d02..e0f0fd525 100644 --- a/plugins/domains/dailymail.co.uk/dailymail.video.js +++ b/plugins/domains/dailymail.co.uk/dailymail.video.js @@ -1,8 +1,10 @@ -const decodeHTML5 = require('entities').decodeHTML5; +import { decodeHTML5 } from 'entities'; +import dailymail_embeddedvideo from './dailymail.embeddedvideo.js'; +import dailymail_galleryvideo from './dailymail.galleryvideo.js'; -module.exports = { +export default { - re: [].concat(require('./dailymail.embeddedvideo').re, require('./dailymail.galleryvideo').re), + re: [].concat(dailymail_embeddedvideo.re, dailymail_galleryvideo.re), provides: 'dailymailVideo', diff --git a/plugins/domains/dailymail.co.uk/mol.im.js b/plugins/domains/dailymail.co.uk/mol.im.js index fed5cf0e5..1a51c2678 100644 --- a/plugins/domains/dailymail.co.uk/mol.im.js +++ b/plugins/domains/dailymail.co.uk/mol.im.js @@ -1,4 +1,4 @@ -module.exports = { +export default { // #v-1467332342001 isn't passed by Dailymail URL shortener re: /^https:\/\/mol\.im\/a\/\d+(#v\-\d+)/i, diff --git a/plugins/domains/dailymotion.com/dailymotion.com.js b/plugins/domains/dailymotion.com/dailymotion.com.js index 57a8b5146..1940d03c6 100644 --- a/plugins/domains/dailymotion.com/dailymotion.com.js +++ b/plugins/domains/dailymotion.com/dailymotion.com.js @@ -1,6 +1,7 @@ -const querystring = require('querystring'); +import * as querystring from 'querystring'; +import request from 'request'; -module.exports = { +export default { mixins: [ "oembed-title", @@ -10,6 +11,7 @@ module.exports = { "domain-icon", "og-description", "canonical", + "oembed-iframe", "video" ], @@ -18,17 +20,15 @@ module.exports = { * - queue-enable=false - https://faq.dailymotion.com/hc/en-us/articles/360000713928-Disabling-the-Up-Next-Queue * - ui-start-screen-info=0 - hide title amontg other things - https://nextgenthemes.com/how-to-hide-titles-and-change-other-setting-for-youtube-vimeo-embeds-in-wordpress-with-arve/ */ - getLink: function (url, oembed, options) { + getLink: function (url, iframe, options) { var playlistParams = querystring.parse(options.getProviderOptions('dailymotion.get_params', '').replace(/^\?/, '')); - var qs = querystring.stringify(playlistParams); - var href = oembed.getIframeAttr('src'); - if (href && oembed.height) { + if (iframe.src && iframe.height) { return { - href: href + (href.indexOf("?") > -1 ? "&" : (qs !== "" ? "?" : "")) + qs, + href: iframe.replaceQuerystring(playlistParams), type: CONFIG.T.text_html, "rel": [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.ssl, CONFIG.R.oembed], - "aspect-ratio": oembed.width / oembed.height, + "aspect-ratio": iframe.width / iframe.height, scrolling: 'no', autoplay: "autoplay=1" }; @@ -49,7 +49,6 @@ module.exports = { tests: [{ getUrls: function(cb) { - var request = require('request'); request({ url: 'https://api.dailymotion.com/videos', json: true diff --git a/plugins/domains/dailymotion.com/dailymotion.playlist.js b/plugins/domains/dailymotion.com/dailymotion.playlist.js index 6f529bc36..b80975b16 100644 --- a/plugins/domains/dailymotion.com/dailymotion.playlist.js +++ b/plugins/domains/dailymotion.com/dailymotion.playlist.js @@ -1,4 +1,4 @@ -module.exports = { +export default { /** * Endpoint `http://www.dailymotion.com/services/oembed` * does not provide oembed results for playlist url currently diff --git a/plugins/domains/dailymotion.com/dailymotion.swf.js b/plugins/domains/dailymotion.com/dailymotion.swf.js index 8335ebedf..55b7a869b 100644 --- a/plugins/domains/dailymotion.com/dailymotion.swf.js +++ b/plugins/domains/dailymotion.com/dailymotion.swf.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.dailymotion\.com\/(swf|embed)\/video\//i, diff --git a/plugins/domains/documentcloud.org.js b/plugins/domains/documentcloud.org.js index eb5610448..c03a3b1ad 100644 --- a/plugins/domains/documentcloud.org.js +++ b/plugins/domains/documentcloud.org.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/(?:www)?\.?documentcloud\.org\/documents?\/\d+/i, diff --git a/plugins/domains/dribbble.com.js b/plugins/domains/dribbble.com.js index a58f818c6..1f5bdd3a3 100644 --- a/plugins/domains/dribbble.com.js +++ b/plugins/domains/dribbble.com.js @@ -1,6 +1,6 @@ const PROFILE_RE = /^https?:\/\/dribbble\.com\/([a-zA-Z0-9\-]+)(?:\?[^\/]+)?$/i; -module.exports = { +export default { re: [ /^https?:\/\/dribbble\.com\/shots\/([a-zA-Z0-9\-]+)/i, diff --git a/plugins/domains/ebaumsworld.com.js b/plugins/domains/ebaumsworld.com.js index 2d5724e59..d998c7b45 100644 --- a/plugins/domains/ebaumsworld.com.js +++ b/plugins/domains/ebaumsworld.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.ebaumsworld\.com\/video\/watch\/(\d+)/i, diff --git a/plugins/domains/espn.com.js b/plugins/domains/espn.com.js index ffb00ceae..bf18f76ae 100644 --- a/plugins/domains/espn.com.js +++ b/plugins/domains/espn.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.)?espn\.com?(?:\.\w{2})?\/video\/clip\?id=espn:(\d+)/i, diff --git a/plugins/domains/facebook.com/facebook.ld.js b/plugins/domains/facebook.com/facebook.ld.js index af0d41371..dbec152eb 100644 --- a/plugins/domains/facebook.com/facebook.ld.js +++ b/plugins/domains/facebook.com/facebook.ld.js @@ -1,6 +1,8 @@ -module.exports = { +import facebook_video_re from './facebook.video.js'; - re: require('./facebook.video').re, +export default { + + re: facebook_video_re.re, getMeta: function(__allowFBThumbnail, schemaVideoObject) { diff --git a/plugins/domains/facebook.com/facebook.meta.js b/plugins/domains/facebook.com/facebook.meta.js index bdaa48d0f..160486128 100644 --- a/plugins/domains/facebook.com/facebook.meta.js +++ b/plugins/domains/facebook.com/facebook.meta.js @@ -1,6 +1,9 @@ -const entities = require('entities'); +import * as entities from 'entities'; -module.exports = { +import facebook_post from './facebook.post.js'; +import facebook_video from './facebook.video.js'; + +export default { /** * HEADS-UP: New endpoints as of Oct 24, 2020: @@ -9,7 +12,7 @@ module.exports = { * as desribed on https://github.com/itteco/iframely/issues/284. */ - re: [].concat(require('./facebook.post').re, require('./facebook.video').re), + re: [].concat(facebook_post.re, facebook_video.re), mixins: [ "domain-icon", diff --git a/plugins/domains/facebook.com/facebook.page.js b/plugins/domains/facebook.com/facebook.page.js index a9f35bf6d..fae39efe6 100644 --- a/plugins/domains/facebook.com/facebook.page.js +++ b/plugins/domains/facebook.com/facebook.page.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(www|m)\.facebook\.com\/([^\/\?]+(?<!\.php))\/?(?:about|photos|videos|events|timeline|photos_stream)?\/?(?:\?[^\/\?]+)?$/i, diff --git a/plugins/domains/facebook.com/facebook.post.js b/plugins/domains/facebook.com/facebook.post.js index 1c9520732..d4a916cc8 100644 --- a/plugins/domains/facebook.com/facebook.post.js +++ b/plugins/domains/facebook.com/facebook.post.js @@ -1,6 +1,6 @@ const DEFAULT_WIDTH = 640; -module.exports = { +export default { re: [ /^https?:\/\/(?:www|m|business)\.facebook\.com\/(permalink|story)\.php\?[^\/]+(\d{10,})/i, diff --git a/plugins/domains/facebook.com/facebook.redirects.js b/plugins/domains/facebook.com/facebook.redirects.js index d24ee0f3d..f531da557 100644 --- a/plugins/domains/facebook.com/facebook.redirects.js +++ b/plugins/domains/facebook.com/facebook.redirects.js @@ -1,6 +1,6 @@ -var URL = require("url"); +import * as URL from "url"; -module.exports = { +export default { re: [ /^https?:\/\/m\.facebook\.com\/story\.php/i, diff --git a/plugins/domains/facebook.com/facebook.thumbnail.js b/plugins/domains/facebook.com/facebook.thumbnail.js index e16d3ee79..dfa4418b3 100644 --- a/plugins/domains/facebook.com/facebook.thumbnail.js +++ b/plugins/domains/facebook.com/facebook.thumbnail.js @@ -1,4 +1,7 @@ -module.exports = { +import facebook_post from './facebook.post.js'; +import facebook_video from './facebook.video.js'; + +export default { provides: "__allowFBThumbnail", @@ -7,7 +10,7 @@ module.exports = { // then we grab meta and get og:image from there if it's not "security checked" for rate limits. // Similar to what we do in domain-icon: ignore if failed. - re: [].concat(require('./facebook.post').re, require('./facebook.video').re), + re: [].concat(facebook_post.re, facebook_video.re), getLink: function(url, __allowFBThumbnail, meta) { diff --git a/plugins/domains/facebook.com/facebook.video.js b/plugins/domains/facebook.com/facebook.video.js index 796bdc535..b0ccbc33f 100644 --- a/plugins/domains/facebook.com/facebook.video.js +++ b/plugins/domains/facebook.com/facebook.video.js @@ -1,6 +1,6 @@ const DEFAULT_WIDTH = 640; -module.exports = { +export default { re: [ /^https?:\/\/(?:www|business)\.facebook\.com\/video\/video\.php.*[\?&]v=(\d{5,})(?:$|&)/i, diff --git a/plugins/domains/flickr.com/flickr.gallery.js b/plugins/domains/flickr.com/flickr.gallery.js index 81b372db6..255924c96 100644 --- a/plugins/domains/flickr.com/flickr.gallery.js +++ b/plugins/domains/flickr.com/flickr.gallery.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.flickr\.com(\/photos\/[@a-zA-Z0-9_\.\-]+\/(?:sets|albums)\/(\d+))/i, diff --git a/plugins/domains/flickr.com/flickr.photo.js b/plugins/domains/flickr.com/flickr.photo.js index 20f176bc5..5df53766d 100644 --- a/plugins/domains/flickr.com/flickr.photo.js +++ b/plugins/domains/flickr.com/flickr.photo.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.flickr\.com\/photos\/([@a-zA-Z0-9_\.\-]+)\/(\d+).*?$/i, diff --git a/plugins/domains/flickr.com/utils.js b/plugins/domains/flickr.com/utils.js index bfd6a708a..7cd7eee1b 100644 --- a/plugins/domains/flickr.com/utils.js +++ b/plugins/domains/flickr.com/utils.js @@ -1,12 +1,10 @@ -(function() { - - var async = require('async'); - var API_URI = 'https://api.flickr.com/services/rest'; - exports.notPlugin = true; + export const notPlugin = true; - exports.getPhotoSizes = function(photo_id, request, api_key, cb) { + // TODO: is this used anywhere? + + export function getPhotoSizes(photo_id, request, api_key, cb) { request({ uri: API_URI, qs: { @@ -22,6 +20,4 @@ cb(error || (body && body.message), body && body.sizes && body.sizes.size); } }, cb); - }; - -})(); \ No newline at end of file + }; \ No newline at end of file diff --git a/plugins/domains/geogebra.iframe.js b/plugins/domains/geogebra.iframe.js index 9b090fe55..119da366d 100644 --- a/plugins/domains/geogebra.iframe.js +++ b/plugins/domains/geogebra.iframe.js @@ -1,16 +1,14 @@ -module.exports = { +export default { re: /^https?:\/\/(?:tube|www)\.geogebra\.org\/material\/\w+\/id\/([a-zA-Z0-9]+)/i, // It's here mostly just to detect proper embed sizing. getLink: function(url, cheerio) { - var $el = cheerio('script'); - var $script = cheerio('script:contains("var parameters =")'); - if ($script.length === 1 && /({.+});/i.test($script.text())) { + if ($script.length === 1 && /({.+});/i.test($script.html())) { try { - var params = JSON.parse ($script.text().match(/({.+});/i)[1]); + var params = JSON.parse ($script.html().match(/({.+});/i)[1]); if (params.width && params.height) { return { diff --git a/plugins/domains/geogebra.org.js b/plugins/domains/geogebra.org.js index ccaefb8e1..7d37a5f3b 100644 --- a/plugins/domains/geogebra.org.js +++ b/plugins/domains/geogebra.org.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(?:tube|www)\.geogebra\.org\/(?:m|classic|graphing)\/(?:[a-zA-Z0-9]+)#material\/([a-zA-Z0-9#]+)/i, diff --git a/plugins/domains/giphy.com.js b/plugins/domains/giphy.com.js index 565428940..58739e6c9 100644 --- a/plugins/domains/giphy.com.js +++ b/plugins/domains/giphy.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/giphy\.com\/(?:gifs|stickers)\/(?:[a-zA-Z0-9_-]+\-)?([a-z0-9\-]+)/i, diff --git a/plugins/domains/github.gist.js b/plugins/domains/github.gist.js index 1d851ca50..4254a8856 100644 --- a/plugins/domains/github.gist.js +++ b/plugins/domains/github.gist.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/gist\.github\.com\/([\w\.\-]+\/)(\w+)(#([\w\.\-]+))?/i, diff --git a/plugins/domains/google.com/accounts.google.com.js b/plugins/domains/google.com/accounts.google.com.js index 19809c071..fed8d9bc8 100644 --- a/plugins/domains/google.com/accounts.google.com.js +++ b/plugins/domains/google.com/accounts.google.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/accounts\.google\.com\/ServiceLogin/i, diff --git a/plugins/domains/google.com/docs.google.com.js b/plugins/domains/google.com/docs.google.com.js index 12f82d77b..67f89853d 100644 --- a/plugins/domains/google.com/docs.google.com.js +++ b/plugins/domains/google.com/docs.google.com.js @@ -1,6 +1,6 @@ -var decodeHTML5 = require('entities').decodeHTML5; +import { decodeHTML5 } from 'entities'; -module.exports = { +export default { provides: "schemaFileObject", diff --git a/plugins/domains/google.com/google.calendar.js b/plugins/domains/google.com/google.calendar.js index 8e94ba830..4f977b7e3 100644 --- a/plugins/domains/google.com/google.calendar.js +++ b/plugins/domains/google.com/google.calendar.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/(?:www|calendar)\.google\.com\/calendar\/embed\/?\?([^\/]+)/i diff --git a/plugins/domains/google.com/google.custommap.js b/plugins/domains/google.com/google.custommap.js index fefaac7fa..3b4da299b 100644 --- a/plugins/domains/google.com/google.custommap.js +++ b/plugins/domains/google.com/google.custommap.js @@ -1,6 +1,6 @@ // Custom Google Maps -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.)?google\.(?:com?\.)?[a-z]+\/maps\/d\/(?:edit|embed|viewer)\?(?:[^&]+&)*(mid)=([a-zA-Z0-9\.\-_]+)/i, diff --git a/plugins/domains/google.com/google.maps.js b/plugins/domains/google.com/google.maps.js index 1e6d8cbd0..dfbfd4a30 100644 --- a/plugins/domains/google.com/google.maps.js +++ b/plugins/domains/google.com/google.maps.js @@ -1,7 +1,7 @@ // Covers new google maps. Not classic ones. Classic ones are handled by maps.google.com.js plugin // Docs are at https://developers.google.com/maps/documentation/embed/guide -module.exports = { +export default { re: [ // place diff --git a/plugins/domains/google.com/google.streetview.js b/plugins/domains/google.com/google.streetview.js index 793a6019c..c0b3af4bb 100644 --- a/plugins/domains/google.com/google.streetview.js +++ b/plugins/domains/google.com/google.streetview.js @@ -1,7 +1,7 @@ // Covers Google Maps street view images only: // https://developers.google.com/maps/documentation/streetview/intro -module.exports = { +export default { re: [ // place diff --git a/plugins/domains/google.com/maps.google.com.js b/plugins/domains/google.com/maps.google.com.js index 6be27faef..4bf5c73c4 100644 --- a/plugins/domains/google.com/maps.google.com.js +++ b/plugins/domains/google.com/maps.google.com.js @@ -1,6 +1,6 @@ -var URL = require("url"); -var _ = require('underscore'); -var QueryString = require("querystring"); +import * as URL from "url"; +import * as _ from 'underscore'; +import * as QueryString from "querystring"; var TypeMap = { m: 'roadmap', @@ -14,7 +14,7 @@ function diameterToZoom (diameter) { return zoom < 0 ? 0 : zoom > 20 ? 20 : zoom; } -module.exports = { +export default { re: [ /^https?:\/\/maps\.google\.(?:com?\.)?[a-z]+\/(?:maps(?:\/ms|\/preview)?)?[\?\#].+/i, diff --git a/plugins/domains/hockeydb.com.js b/plugins/domains/hockeydb.com.js index 625c21101..789a1332f 100644 --- a/plugins/domains/hockeydb.com.js +++ b/plugins/domains/hockeydb.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.hockeydb\.com\/ihdb\/stats\/pdisplay\.php\?pid=(\d+)/i, diff --git a/plugins/domains/i.gfycat.com.js b/plugins/domains/i.gfycat.com.js index 24fa38f4d..2807de952 100644 --- a/plugins/domains/i.gfycat.com.js +++ b/plugins/domains/i.gfycat.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/(?:giant|thumbs|zippy)\.gfycat\.com\/([a-zA-Z0-9]+)(?:\-mobile|\-size_restricted)?\.(?:webm|mp4|gif)$/i, diff --git a/plugins/domains/i.gifs.com.js b/plugins/domains/i.gifs.com.js index 03271791e..aa9fb684e 100644 --- a/plugins/domains/i.gifs.com.js +++ b/plugins/domains/i.gifs.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/j\.gifs\.com\/(\w+)\.gif$/i diff --git a/plugins/domains/i.giphy.com.js b/plugins/domains/i.giphy.com.js index 103c0bea9..c4d01fa1a 100644 --- a/plugins/domains/i.giphy.com.js +++ b/plugins/domains/i.giphy.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/i\.giphy\.com\/(\w+)\.gif(\?.*)?$/i, diff --git a/plugins/domains/ignore.js b/plugins/domains/ignore.js index 0a0bbb944..3eb30c8fb 100644 --- a/plugins/domains/ignore.js +++ b/plugins/domains/ignore.js @@ -2,7 +2,7 @@ const RE = CONFIG.IGNORE_DOMAINS_RE || CONFIG.BLACKLIST_DOMAINS_RE; -module.exports = { +export default { re: RE, diff --git a/plugins/domains/imageshack.com.js b/plugins/domains/imageshack.com.js index ada5358dd..7e7e957d5 100644 --- a/plugins/domains/imageshack.com.js +++ b/plugins/domains/imageshack.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/imageshack\.com\/i\/\w+/i, mixins: [ diff --git a/plugins/domains/imdb.com.js b/plugins/domains/imdb.com.js index ac1f7990c..d067c69b6 100644 --- a/plugins/domains/imdb.com.js +++ b/plugins/domains/imdb.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.imdb\.com\/video\/(?:[\w]+\/)?vi(\d+)/i, diff --git a/plugins/domains/instagram.com/instagram-post-allow-meta.js b/plugins/domains/instagram.com/instagram-post-allow-meta.js index 662027d18..87f7dfc19 100644 --- a/plugins/domains/instagram.com/instagram-post-allow-meta.js +++ b/plugins/domains/instagram.com/instagram-post-allow-meta.js @@ -1,6 +1,8 @@ -module.exports = { +import instagram_com from './instagram.com.js'; - re: require('./instagram.com').re, +export default { + + re: instagram_com.re, provides: 'ipOG', diff --git a/plugins/domains/instagram.com/instagram-post-rate-limit-429.js b/plugins/domains/instagram.com/instagram-post-rate-limit-429.js index 003fca3ba..1c904d1a4 100644 --- a/plugins/domains/instagram.com/instagram-post-rate-limit-429.js +++ b/plugins/domains/instagram.com/instagram-post-rate-limit-429.js @@ -1,6 +1,8 @@ -module.exports = { +import instagram_com from './instagram.com.js'; - re: require('./instagram.com').re, +export default { + + re: instagram_com.re, provides: 'ipOG', diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js index f6c041973..63eeb4849 100644 --- a/plugins/domains/instagram.com/instagram.com.js +++ b/plugins/domains/instagram.com/instagram.com.js @@ -1,7 +1,8 @@ -const cheerio = require('cheerio'); -const decodeHTML5 = require('entities').decodeHTML5; +import cheerio from 'cheerio'; -module.exports = { +import { decodeHTML5 } from 'entities'; + +export default { /** * HEADS-UP: New endpoints as of Oct 24, 2020: diff --git a/plugins/domains/issuu.com.js b/plugins/domains/issuu.com.js index 33c43532f..7afa45739 100644 --- a/plugins/domains/issuu.com.js +++ b/plugins/domains/issuu.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/issuu\.com\/[\w_.-]+\/docs\/([\w_.-]+)/i, diff --git a/plugins/domains/itunes.apple.com/apple.music.js b/plugins/domains/itunes.apple.com/apple.music.js index 1452b5a18..a13935c51 100644 --- a/plugins/domains/itunes.apple.com/apple.music.js +++ b/plugins/domains/itunes.apple.com/apple.music.js @@ -1,7 +1,7 @@ -const URL = require('url'); -const _ = require('underscore'); +import * as URL from 'url'; +import * as _ from 'underscore'; -module.exports = { +export default { re: [ /^https?:\/\/music\.apple\.com\/(\w{2})\/(album)(?:\/[^\/]+)?\/id(\d+)\?i=(\d+)?/i, diff --git a/plugins/domains/jsfiddle.net.js b/plugins/domains/jsfiddle.net.js index 6dfbe6e2d..85048acda 100644 --- a/plugins/domains/jsfiddle.net.js +++ b/plugins/domains/jsfiddle.net.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^(https?:\/\/jsfiddle.net\/(?:\w+\/)?\w+\/)\/?(?:[^\/]+)?$/i, diff --git a/plugins/domains/kickstarter.com.js b/plugins/domains/kickstarter.com.js index 5958ec936..61ab356ec 100644 --- a/plugins/domains/kickstarter.com.js +++ b/plugins/domains/kickstarter.com.js @@ -1,6 +1,6 @@ -var utils = require('../../lib/utils'); +import * as utils from '../../lib/utils.js'; -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.)?kickstarter\.com\/projects\/[a-zA-Z0-9-]+\/[a-zA-Z0-9-]+\/?(?:widget\/video\.html)?(?:\?.*)?$/i diff --git a/plugins/domains/knightlab.js b/plugins/domains/knightlab.js index 7d16c9289..2e152a53e 100644 --- a/plugins/domains/knightlab.js +++ b/plugins/domains/knightlab.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/s3\.amazonaws\.com\/(?:uploads|CDN)\.knightlab\.com\/(storymapjs)\/\w+\/(?:[a-zA-Z0-9\-\/]+)\.html/i, diff --git a/plugins/domains/libsyn.com.js b/plugins/domains/libsyn.com.js index 6def89e57..67c9274ae 100644 --- a/plugins/domains/libsyn.com.js +++ b/plugins/domains/libsyn.com.js @@ -1,10 +1,9 @@ -module.exports = { +export default { re: /^https?:\/\/[a-zA-Z0-9\.]+\.libsyn(?:pro)?\.com\//, mixins: [ - "*", - "oembed-iframe" + "*" ], getLink: function(iframe, meta, options) { diff --git a/plugins/domains/live.amcharts.com.js b/plugins/domains/live.amcharts.com.js index 8f08b7890..0f8828f69 100644 --- a/plugins/domains/live.amcharts.com.js +++ b/plugins/domains/live.amcharts.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/live\.amcharts\.com\/([a-zA-Z0-9\-]+)/i, diff --git a/plugins/domains/livestream.com.js b/plugins/domains/livestream.com.js index 2fdb6ed67..80f6efa56 100644 --- a/plugins/domains/livestream.com.js +++ b/plugins/domains/livestream.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/livestream\.com\/(\w+)\/events\/(\d+)(\/videos?\/\d+)?/i, diff --git a/plugins/domains/mail.ru.js b/plugins/domains/mail.ru.js index eb665bc0e..89cfe0993 100644 --- a/plugins/domains/mail.ru.js +++ b/plugins/domains/mail.ru.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/my\.mail\.ru\/\/?[a-z\.]+\/[a-zA-Z0-9\._\-]+\/video\/([a-zA-Z0-9_]+)\/([a-zA-Z0-9_]+)\.html/i diff --git a/plugins/domains/maps.yandex.ru/maps.yandex.ru.js b/plugins/domains/maps.yandex.ru/maps.yandex.ru.js index 345e0ee47..0c781433f 100644 --- a/plugins/domains/maps.yandex.ru/maps.yandex.ru.js +++ b/plugins/domains/maps.yandex.ru/maps.yandex.ru.js @@ -1,6 +1,6 @@ -var URL = require("url"); +import * as URL from "url"; -module.exports = { +export default { re: /^https:\/\/yandex\.ru\/maps\//, diff --git a/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js b/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js index 9641ff518..5225db92d 100644 --- a/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js +++ b/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https:\/\/yandex\.ru\/maps\/-\//, diff --git a/plugins/domains/medium.com.js b/plugins/domains/medium.com.js index 663f58d76..bf1ba5d1e 100644 --- a/plugins/domains/medium.com.js +++ b/plugins/domains/medium.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https:\/\/(?:[a-z0-9\-]+\.)?medium\.com\/@?[\w-]+/i, diff --git a/plugins/domains/mixcloud.com.js b/plugins/domains/mixcloud.com.js index 565ad6302..67fb909ba 100644 --- a/plugins/domains/mixcloud.com.js +++ b/plugins/domains/mixcloud.com.js @@ -1,8 +1,4 @@ -const cheerio = require('cheerio'); -const querystring = require('querystring'); -const URL = require("url"); - -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.)mixcloud\.com\/(?:live\/)?[a-zA-Z0-9\.\-_]+\/?(?:\?.+)?$/, @@ -15,13 +11,14 @@ module.exports = { "oembed-author", "oembed-site", "oembed-error", + "oembed-iframe", "domain-icon" ], - getLink: function (oembed, whitelistRecord, options) { + getLink: function (oembed, iframe, whitelistRecord, options) { // let Whitelist/default fallbacks take control if oEmbed fails - if (!(oembed.type === "rich" && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.rich'))) { + if (!(oembed.type === "rich" && iframe.src && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.rich'))) { return; } else { @@ -30,92 +27,82 @@ module.exports = { type: CONFIG.T.text_html }; - var $container = cheerio('<div>'); - try { - $container.html(oembed.html); - } catch (ex) {} - - var $iframe = $container.find('iframe'); - - if ($iframe.length == 1) { - - var href = $iframe.attr('src'); - - if (/\/widget\/follow\//.test(href)) { - widget.href = href; - widget.rel.push(CONFIG.R.summary); - widget.width = oembed.width; - widget.height = oembed.height; - - // widget.error = 'Mixcloud user summary is currently broken'; // Sept 22, 2020 - works as of Jan 25, 2021 - - } else { - - var params = URL.parse(href, true).query; - if (options.getProviderOptions('players.horizontal') === false) { - delete params.hide_cover; - } - var style = options.getRequestOptions('mixcloud.style', params.mini == 1 ? 'mini' : (params.hide_cover == 1 ? 'classic' : 'cover')); - var theme = options.getRequestOptions('players.theme', params.light == 1 ? 'light' : 'dark'); - - if (theme === 'light') { - params.light = 1; - } - - if (options.getRequestOptions('mixcloud.hide_artwork', params.hide_artwork)) { - params.hide_artwork = 1; - } - - if (style === 'mini') { - params.mini = 1; - params.hide_cover = 1; - } else if (style === 'classic') { - delete params.mini; - params.hide_cover = 1; - } else if (style === 'cover') { - delete params.mini; - delete params.hide_cover; - delete params.light; - delete params.hide_artwork; + var href = iframe.src; + + if (/\/widget\/follow\//.test(href)) { + widget.href = href; + widget.rel.push(CONFIG.R.summary); + widget.width = oembed.width; + widget.height = oembed.height; + + // widget.error = 'Mixcloud user summary is currently broken'; // Sept 22, 2020 - works as of Jan 25, 2021 + + } else { + + var params = Object.assign(iframe.query); + if (options.getProviderOptions('players.horizontal') === false) { + delete params.hide_cover; + } + var style = options.getRequestOptions('mixcloud.style', params.mini == 1 ? 'mini' : (params.hide_cover == 1 ? 'classic' : 'cover')); + var theme = options.getRequestOptions('players.theme', params.light == 1 ? 'light' : 'dark'); + + if (theme === 'light') { + params.light = 1; + } + + if (options.getRequestOptions('mixcloud.hide_artwork', params.hide_artwork)) { + params.hide_artwork = 1; + } + + if (style === 'mini') { + params.mini = 1; + params.hide_cover = 1; + } else if (style === 'classic') { + delete params.mini; + params.hide_cover = 1; + } else if (style === 'cover') { + delete params.mini; + delete params.hide_cover; + delete params.light; + delete params.hide_artwork; + } + + widget.href = iframe.assignQuerystring(params); + widget.autoplay = 'autoplay=1'; + + // mixcloud ignores &mini=1 if there's no &hide_cover=1. + widget.height = !/&?hide_cover=1/i.test(widget.href) ? 400 : (/&?mini=1/i.test(widget.href) ? 60 : 120); + widget.scrolling = "no"; + widget.rel.push(CONFIG.R.player); + widget.rel.push(CONFIG.R.auido); + + widget.options = { + style: { + label: 'Widget style', + value: style, + values: { + 'mini': 'Mini', + 'classic': 'Classic', + 'cover': 'Picture' + } } - - widget.href = href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1'); - widget.autoplay = 'autoplay=1'; - - // mixcloud ignores &mini=1 if there's no &hide_cover=1. - widget.height = !/&?hide_cover=1/i.test(widget.href) ? 400 : (/&?mini=1/i.test(widget.href) ? 60 : 120); - widget.scrolling = "no"; - widget.rel.push(CONFIG.R.player); - widget.rel.push(CONFIG.R.auido); - - widget.options = { - style: { - label: 'Widget style', - value: style, - values: { - 'mini': 'Mini', - 'classic': 'Classic', - 'cover': 'Picture' - } + }; + + if (style !== 'cover') { + widget.options.theme = { + label: CONFIG.L.theme, + value: theme, + values: { + light: CONFIG.L.light, + dark: CONFIG.L.dark } }; + widget.options.hide_artwork = { + label: CONFIG.L.hide_artwork, + value: params.hide_artwork === 1 + }; - if (style !== 'cover') { - widget.options.theme = { - label: CONFIG.L.theme, - value: theme, - values: { - light: CONFIG.L.light, - dark: CONFIG.L.dark - } - }; - widget.options.hide_artwork = { - label: CONFIG.L.hide_artwork, - value: params.hide_artwork === 1 - }; - - } - } + } return [widget, { href: oembed.image, diff --git a/plugins/domains/momindum.com.js b/plugins/domains/momindum.com.js index 13b441407..2dbc2e334 100644 --- a/plugins/domains/momindum.com.js +++ b/plugins/domains/momindum.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { mixins: [ "*" diff --git a/plugins/domains/nbcnews.com.js b/plugins/domains/nbcnews.com.js index 74e1db79a..658b7cfa4 100644 --- a/plugins/domains/nbcnews.com.js +++ b/plugins/domains/nbcnews.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.nbcnews\.com\/(?:[a-z\-]+\/)?videos?\/[a-zA-Z0-9-]+\-(\d+)/i, diff --git a/plugins/domains/nbcsports.com.js b/plugins/domains/nbcsports.com.js index 5387d4e0d..000020ba5 100644 --- a/plugins/domains/nbcsports.com.js +++ b/plugins/domains/nbcsports.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.nbcsports\.com\/videos?\/[a-zA-Z0-9-]+/i diff --git a/plugins/domains/nhl.com/nhl.com.js b/plugins/domains/nhl.com/nhl.com.js index 2e79997e8..458569ad6 100644 --- a/plugins/domains/nhl.com/nhl.com.js +++ b/plugins/domains/nhl.com/nhl.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/(?:www\.)?nhl\.com(\/\w+\/)video\/(?:embed\/)?([a-zA-Z0-9\-]+\/t\-\d+\/c\-\d+)/i, diff --git a/plugins/domains/npr.org/npr.sections.js b/plugins/domains/npr.org/npr.sections.js index e79882272..b2fa55989 100644 --- a/plugins/domains/npr.org/npr.sections.js +++ b/plugins/domains/npr.org/npr.sections.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.npr\.org\/sections\//i, diff --git a/plugins/domains/nytimes.video.js b/plugins/domains/nytimes.video.js index 01a875c51..bda5ec89d 100644 --- a/plugins/domains/nytimes.video.js +++ b/plugins/domains/nytimes.video.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.nytimes\.com\/video\/[a-zA-Z0-9\-]+\/(\d+)\//, diff --git a/plugins/domains/openstreetmap.org.js b/plugins/domains/openstreetmap.org.js index 43cb9f47a..59a6f9aef 100644 --- a/plugins/domains/openstreetmap.org.js +++ b/plugins/domains/openstreetmap.org.js @@ -1,5 +1,5 @@ -var URL = require("url"); -var QueryString = require("querystring"); +import * as URL from "url"; +import * as QueryString from "querystring"; var LayerMap = { M: 'mapnik', @@ -53,7 +53,7 @@ function getBBox(lat, lon, zoom, width, height) { return [lonlat_s[0], lonlat_s[1], lonlat_e[0], lonlat_e[1]]; } -module.exports = { +export default { re: /^https?:\/\/(?:www\.)?openstreetmap\.org\/(?:node\/\d+)?(?:\?.+|\#.*map=.+|export\/embed\.html\?)/i, diff --git a/plugins/domains/ow.ly.js b/plugins/domains/ow.ly.js index 27e6b6dce..da63cbd82 100644 --- a/plugins/domains/ow.ly.js +++ b/plugins/domains/ow.ly.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/ow\.ly\/i\//i diff --git a/plugins/domains/padlet.com.js b/plugins/domains/padlet.com.js index 910f80db3..88adbb2f8 100644 --- a/plugins/domains/padlet.com.js +++ b/plugins/domains/padlet.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/padlet\.com\/[0-9a-zA-Z_\-]+\/([0-9a-zA-Z_\-]+)/i diff --git a/plugins/domains/pastebin.com.js b/plugins/domains/pastebin.com.js index d55669639..d08c8c12c 100644 --- a/plugins/domains/pastebin.com.js +++ b/plugins/domains/pastebin.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/pastebin\.com\/(?!search)([a-zA-Z0-9]+)/i diff --git a/plugins/domains/piktochart.com/piktochart.com.js b/plugins/domains/piktochart.com/piktochart.com.js index e6efc9f66..2e954a64d 100644 --- a/plugins/domains/piktochart.com/piktochart.com.js +++ b/plugins/domains/piktochart.com/piktochart.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/(?:magic|create)\.piktochart\.com\/output\/(\d+\-[\-a-zA-Z-0-9_]+)/i, diff --git a/plugins/domains/pinterest.com/pinterest.board.js b/plugins/domains/pinterest.com/pinterest.board.js index 158d7d500..4f4bd8f5e 100644 --- a/plugins/domains/pinterest.com/pinterest.board.js +++ b/plugins/domains/pinterest.com/pinterest.board.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/(?:\w{2,3}\.)?pinterest(?:\.com?)?\.\w{2,3}\/((?!pin)[a-zA-Z0-9%_]+|pinterest)\/([a-zA-Z0-9%\-]+)\/?(?:$|\?|#)/i, diff --git a/plugins/domains/pinterest.com/pinterest.pin.js b/plugins/domains/pinterest.com/pinterest.pin.js index 56def9f4d..e1f67efc0 100644 --- a/plugins/domains/pinterest.com/pinterest.pin.js +++ b/plugins/domains/pinterest.com/pinterest.pin.js @@ -1,11 +1,10 @@ -module.exports = { +export default { re: /^(https?:\/\/(?:\w{2,3}\.)?pinterest(?:\.com?)?\.\w{2,3})\/pin\/(?:[^\/]+\-)?(\d+)/i, mixins: [ - "*", - "oembed-iframe" + "*" ], // https://developers.pinterest.com/tools/widget-builder/?type=pin&terse=true&size=large diff --git a/plugins/domains/pinterest.com/pinterest.user.js b/plugins/domains/pinterest.com/pinterest.user.js index bafa21366..266546af1 100644 --- a/plugins/domains/pinterest.com/pinterest.user.js +++ b/plugins/domains/pinterest.com/pinterest.user.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/(?:\w{2,3}\.)?pinterest(?:\.com?)?\.\w{2,3}\/((?!pin)[a-zA-Z0-9%_]+|pinterest)\/?(?:$|\?|#)/i, diff --git a/plugins/domains/polldaddy.com.js b/plugins/domains/polldaddy.com.js index b737898f8..f534adb10 100644 --- a/plugins/domains/polldaddy.com.js +++ b/plugins/domains/polldaddy.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(?:\w+\.)?polldaddy\.com\/poll\/([0-9]+)/i, diff --git a/plugins/domains/prezi.com.js b/plugins/domains/prezi.com.js index cb460d533..dc1f6fd3c 100644 --- a/plugins/domains/prezi.com.js +++ b/plugins/domains/prezi.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https:\/\/prezi\.com\/([a-z0-9\-_]+)\/[^\/]+\/(?:\?[^\/\?]+)?$/i, diff --git a/plugins/domains/quizlet.com.js b/plugins/domains/quizlet.com.js index b29a759fb..d4dc4cf03 100644 --- a/plugins/domains/quizlet.com.js +++ b/plugins/domains/quizlet.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/quizlet\.com\/(?:\w{2}\/)?(\d+)\/([^\/]+)\/?/i diff --git a/plugins/domains/rapgenius.com.js b/plugins/domains/rapgenius.com.js index 7b469de82..f90770575 100644 --- a/plugins/domains/rapgenius.com.js +++ b/plugins/domains/rapgenius.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/(?:[\w\-]+\.)?genius\.com\/(?!jobs)([a-zA-Z0-9\-]+)/i, diff --git a/plugins/domains/readymag.com.js b/plugins/domains/readymag.com.js index 773912e5d..563c9b702 100644 --- a/plugins/domains/readymag.com.js +++ b/plugins/domains/readymag.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/readymag\.com\/\w+\/(\d+)/i diff --git a/plugins/domains/scribd.com/scribd.com-error.js b/plugins/domains/scribd.com/scribd.com-error.js index 74ae24214..e155a89e7 100644 --- a/plugins/domains/scribd.com/scribd.com-error.js +++ b/plugins/domains/scribd.com/scribd.com-error.js @@ -1,8 +1,9 @@ -const URL = require("url"); +import * as URL from "url"; +import scribd_com from './scribd.com.js'; -module.exports = { +export default { - re: require('./scribd.com').re, + re: scribd_com.re, provides: ["scribdData"], diff --git a/plugins/domains/scribd.com/scribd.com.js b/plugins/domains/scribd.com/scribd.com.js index dffc16ab7..fd53f1be1 100644 --- a/plugins/domains/scribd.com/scribd.com.js +++ b/plugins/domains/scribd.com/scribd.com.js @@ -1,22 +1,14 @@ -const $ = require('cheerio'); -const utils = require('../../../lib/utils'); -const querystring = require('querystring'); -const URL = require("url"); - -module.exports = { +export default { re: [ /^https?:\/\/(?:www|\w{2})\.scribd\.com\/(doc|document|embeds|presentation|fullscreen)\/(\d+)/i ], - provides: ['scribdData'], - - mixins: [ "*" ], + mixins: [ "*", "query"], - getLink: function(url, scribdData, options) { - var href = scribdData.href; - var params = URL.parse(href, true).query; - var hash = URL.parse(url, true).hash; + getLink: function(url, iframe, query, options) { + var params = Object.assign(iframe.query); + var hash = query.hash; var slideshow = options.getRequestOptions('scribd.slideshow', params.view_mode === 'slideshow'); if (slideshow) { @@ -36,10 +28,10 @@ module.exports = { } return { - href: href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1'), + href: iframe.assignQuerystring(params), accept: CONFIG.T.text_html, rel: slideshow ? [CONFIG.R.player, CONFIG.R.slideshow, CONFIG.R.html5, CONFIG.R.oembed] : [CONFIG.R.reader, CONFIG.R.html5, CONFIG.R.oembed], - 'aspect-ratio': scribdData.aspect, + 'aspect-ratio': iframe['data-aspect-ratio'], 'padding-bottom': 45, // toolbar options: { slideshow: { @@ -55,48 +47,15 @@ module.exports = { } }, - getData: function(urlMatch, og, oembed, options, cb) { - - if (!og.image) { - return 'embeds' === urlMatch[1] - ? cb({redirect: `https://www.scribd.com/document/${urlMatch[2]}`}) - : cb(null, null); - } - - utils.getImageMetadata(og.image.value || og.image, options, function(error, data) { - - if (error || data.error) { - console.log ('Error getting preview for Scribd: ' + error); - } else { - var $container = $('<div>'); - try { - $container.html(oembed.html); - } catch(ex) {} - - var $iframe = $container.find('iframe'); - if ($iframe.length === 1) { - - return cb(null, { - scribdData: { - aspect: - data.width - && data.height - ? data.width / data.height - : (oembed.thumbnail_height ? oembed.thumbnail_width / oembed.thumbnail_height : null), - - href: $iframe.attr('src') - } - }) - - } else { - return cb(null, null) - } - } - }); + getData: function(urlMatch, options, cb) { + return 'embeds' === urlMatch[1] + ? cb({redirect: `https://www.scribd.com/document/${urlMatch[2]}`}) + : cb(null, null); }, tests: [{ - noFeeds: true + noFeeds: true, + skipMethods: ['getData'] }, "https://www.scribd.com/doc/116154615/Australia-Council-Arts-Funding-Guide-2013", "https://www.scribd.com/document/399637688/Prestons-Advert" diff --git a/plugins/domains/simplecast.com.js b/plugins/domains/simplecast.com.js index a2c5ddcb8..dce05bbf9 100644 --- a/plugins/domains/simplecast.com.js +++ b/plugins/domains/simplecast.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { // also used for hosted simplecasts re: [ diff --git a/plugins/domains/slid.es.js b/plugins/domains/slid.es.js index 2e8af5342..3b35555d2 100644 --- a/plugins/domains/slid.es.js +++ b/plugins/domains/slid.es.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/slides\.com\/([a-zA-Z0-9_\-]+)\/([a-zA-Z0-9_\-]+)/i ], diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js index 9d401bb12..5ac94e6c9 100644 --- a/plugins/domains/slideshare.net.js +++ b/plugins/domains/slideshare.net.js @@ -1,7 +1,6 @@ -var utils = require('../../lib/utils'); -var $ = require('cheerio'); +import * as utils from '../../lib/utils.js'; -module.exports = { +export default { mixins: [ // "*" // Linking to * will enable oembed-rich and will result in incorrect aspect-ratios @@ -12,7 +11,8 @@ module.exports = { "canonical", "description", "oembed-site", - "oembed-title" + "oembed-title", + "oembed-iframe" ], getMeta: function(meta) { @@ -30,62 +30,47 @@ module.exports = { }, - getLink: function(oembed, options, cb) { + getLink: function(oembed, iframe, options, cb) { - if (oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) { - var links = []; + if (iframe.src && oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) { var firstSlide = (/^\/\//.test(oembed.slide_image_baseurl) ? 'http:' : '') + oembed.slide_image_baseurl + '1' + oembed.slide_image_baseurl_suffix; utils.getImageMetadata(firstSlide, options, function(error, data) { - if (error || data.error) { - - console.log ('Error getting first slide for Slideshare: ' + error); - - } else if (data.width && data.height) { - - links.push({ - href: firstSlide, - type: CONFIG.T.image, - rel: CONFIG.R.thumbnail, - width: data.width, - height: data.height - }); - } - - var $container = $('<div>'); - try { - $container.html(oembed.html); - } catch(ex) {} - - var $iframe = $container.find('iframe'); - - var aspect = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height; - - if ($iframe.length == 1) { - links.push({ - href: $iframe.attr('src').replace('http:', ''), - type: CONFIG.T.text_html, - rel: [aspect > 1 ? CONFIG.R.player : CONFIG.R.reader, CONFIG.R.slideshow, CONFIG.R.html5], - "aspect-ratio": aspect, - "padding-bottom": 58 - }); + if (error || data.error || !data.width || !data.height) { + + return cb('Error getting first slide for Slideshare: ' + error); + + } else { + + var aspect = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height; + + return cb(null, [{ + href: firstSlide, + type: CONFIG.T.image, + rel: CONFIG.R.thumbnail, + width: data.width, + height: data.height + }, { + href: oembed.thumbnail, + type: CONFIG.T.image, + rel: [CONFIG.R.thumbnail, CONFIG.R.oembed], + width: oembed.thumbnail_width, + height: data.height ? Math.round (oembed.thumbnail_width / (data.width / data.height)) : oembed.thumbnail_height + }, { + href: iframe.src, + type: CONFIG.T.text_html, + rel: [aspect > 1 ? CONFIG.R.player : CONFIG.R.reader, CONFIG.R.slideshow, CONFIG.R.html5], + "aspect-ratio": aspect, + "padding-bottom": 58 + } + ]); } - links.push ({ - href: oembed.thumbnail, - type: CONFIG.T.image, - rel: [CONFIG.R.thumbnail, CONFIG.R.oembed], - width: oembed.thumbnail_width, - height: data.height ? Math.round (oembed.thumbnail_width / (data.width / data.height)) : oembed.thumbnail_height - }); - - cb(null, links); - }); } else { - cb (null, null); + cb(null, null); } }, @@ -104,7 +89,7 @@ module.exports = { page: "http://www.slideshare.net/popular/today", selector: "a.iso_slideshow_link" }, {skipMethods: ["getData"]}, - "http://www.slideshare.net/geniusworks/gamechangers-the-next-generation-of-business-innovation-by-peter-fisk#btnNext", + "https://www.slideshare.net/DataReportal/digital-2020-global-digital-overview-january-2020-v01-226017535", "https://www.slideshare.net/EnjoyDigitAll/le-design-thinking-by-enjoydigitall-71136562" ] }; \ No newline at end of file diff --git a/plugins/domains/smugmug.com.js b/plugins/domains/smugmug.com.js index f67ac9f47..3d4f2cdc2 100644 --- a/plugins/domains/smugmug.com.js +++ b/plugins/domains/smugmug.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { mixins: [ "*" diff --git a/plugins/domains/soundcloud.com/soundcloud-fallbacks.js b/plugins/domains/soundcloud.com/soundcloud-fallbacks.js index 1c1f69a2d..46dde6e9a 100644 --- a/plugins/domains/soundcloud.com/soundcloud-fallbacks.js +++ b/plugins/domains/soundcloud.com/soundcloud-fallbacks.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: ['sound'], diff --git a/plugins/domains/soundcloud.com/soundcloud-oembed-error.js b/plugins/domains/soundcloud.com/soundcloud-oembed-error.js index 38a2cc529..c4beb1ce9 100644 --- a/plugins/domains/soundcloud.com/soundcloud-oembed-error.js +++ b/plugins/domains/soundcloud.com/soundcloud-oembed-error.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: ['__allow_soundcloud_meta', 'iframe'], diff --git a/plugins/domains/soundcloud.com/soundcloud.com.js b/plugins/domains/soundcloud.com/soundcloud.com.js index c6d11a2e6..a27734e83 100644 --- a/plugins/domains/soundcloud.com/soundcloud.com.js +++ b/plugins/domains/soundcloud.com/soundcloud.com.js @@ -1,16 +1,13 @@ -const $ = require('cheerio'); -const querystring = require('querystring'); -const URL = require("url"); +export default { -module.exports = { - - provides: ['__allow_soundcloud_meta', 'sound', 'iframe'], + provides: ['__allow_soundcloud_meta', 'sound'], mixins: [ "oembed-title", "oembed-site", "oembed-author", "oembed-description", + "oembed-iframe", // do not link to meta as it disables support for direct player urls redirects from w.soundcloud.com "domain-icon" ], @@ -21,8 +18,7 @@ module.exports = { if (iframe.src && sound.count !== 0) { - var href = iframe.src; - var params = URL.parse(href, true).query; + var params = Object.assign(iframe.query); if (options.getRequestOptions('players.horizontal', options.getProviderOptions('soundcloud.old_player') || options.getProviderOptions(CONFIG.O.less))) { params.visual = false; @@ -37,7 +33,7 @@ module.exports = { params.color = options.getProviderOptions('soundcloud.color'); } - href = href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1'); + var href = iframe.assignQuerystring(params); var height = options.getRequestOptions('soundcloud.height', options.getProviderOptions('players.horizontal') === false ? 0 : (/visual=false/.test(href) ? 166 : iframe.height)); // Fallback to old values. if (height === 'auto') { @@ -118,8 +114,7 @@ module.exports = { ) || !oembed.description) ) { return { - __allow_soundcloud_meta: true, - iframe: oembed.getIframe() + __allow_soundcloud_meta: true } } else { @@ -132,8 +127,7 @@ module.exports = { width: oembed.thumbnail_width, height: oembed.thumbnail_height } - }, - iframe: oembed.getIframe() + } } } }, diff --git a/plugins/domains/soundcloud.com/w.soundcloud.com.js b/plugins/domains/soundcloud.com/w.soundcloud.com.js index bd8851bae..0cbb93820 100644 --- a/plugins/domains/soundcloud.com/w.soundcloud.com.js +++ b/plugins/domains/soundcloud.com/w.soundcloud.com.js @@ -1,6 +1,6 @@ -const URL = require('url'); +import * as URL from 'url'; -module.exports = { +export default { re: [ /^https:?\/\/w\.soundcloud\.com\/player\/?\?/i diff --git a/plugins/domains/speakerdeck.com.js b/plugins/domains/speakerdeck.com.js index a36ffb5ea..c732438b1 100644 --- a/plugins/domains/speakerdeck.com.js +++ b/plugins/domains/speakerdeck.com.js @@ -1,53 +1,32 @@ -var $ = require('cheerio'); - -module.exports = { +export default { mixins: [ - "oembed-title", - "oembed-site", - "oembed-author", - "domain-icon", - "og-description" + "*", "query" ], - getLink: function (url, oembed) { - var $container = $('<div>'); - try { - $container.html(oembed.html); - } catch(ex) {} + getLink: function (url, iframe, query, options) { - var $iframe = $container.find('iframe'); - var doc; + var slide = options.getRequestOptions('speakerdeck.slide', query.slide ? parseInt(query.slide) || 1 : 1); - if ($iframe.length == 1) { - var href = $iframe.attr('src').replace(/^\/\//, 'https://'); - if (/\?slide=\d+/i.test(url)) { + if (iframe.src && iframe.width && iframe.height) { + var href = iframe.src.replace(/^\/\//, 'https://'); + if (slide > 1) { href += href.indexOf('?') > -1 ? '&' : '?'; - href += url.match(/\?(slide=\d+)/i)[1]; + href += 'slide=' + slide; } - doc = { + return { href: href, type: CONFIG.T.text_html, rel: [CONFIG.R.player, CONFIG.R.html5], - "aspect-ratio": oembed.width / oembed.height + "aspect-ratio": iframe.width / iframe.height, + options: { + slide: { + label: CONFIG.L.page, + value: slide + } + } } } - - var thumbnail; - - if (doc) { - var id = doc.href.match(/\/\/speakerdeck\.com\/player\/([a-z0-9\-]+)/i)[1]; - - if (id) { - thumbnail = { - href: 'https://speakerd.s3.amazonaws.com/presentations/' + id + '/slide_0.jpg', - type: CONFIG.T.image, - rel: [CONFIG.R.thumbnail] - }; - } - } - - return [doc, thumbnail]; }, diff --git a/plugins/domains/spotify.com-meta-fallback.js b/plugins/domains/spotify.com-meta-fallback.js index cad168e23..b445f041f 100644 --- a/plugins/domains/spotify.com-meta-fallback.js +++ b/plugins/domains/spotify.com-meta-fallback.js @@ -1,6 +1,8 @@ -module.exports = { +import spotify_com from './spotify.com.js'; - re: require('./spotify.com').re, +export default { + + re: spotify_com.re, provides: ['meta'], diff --git a/plugins/domains/spotify.com.js b/plugins/domains/spotify.com.js index f11d9fca6..2b7adf0bf 100644 --- a/plugins/domains/spotify.com.js +++ b/plugins/domains/spotify.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(?:open|play|www)\.spotify\.com\/(?:track|album|artist|show|episode|playlist)/i @@ -6,9 +6,9 @@ module.exports = { mixins: [ "oembed-title", - "oembed-iframe", "og-image", "oembed-thumbnail", + "oembed-iframe", "domain-icon" ], @@ -28,18 +28,16 @@ module.exports = { if (iframe.src) { - var src = iframe.src; - var horizontal_player = options.getRequestOptions('players.horizontal', options.getProviderOptions(CONFIG.O.less)); var player = { - href: src, + href: iframe.src, type: CONFIG.T.text_html, rel: [CONFIG.R.player, CONFIG.R.ssl, CONFIG.R.html5], options: {} }; - if (/album|playlist/.test(src)) { + if (/album|playlist/.test(iframe.src)) { var include_playlist = options.getRequestOptions('spotify.playlist', true); player.rel.push(CONFIG.R.playlist); player.options.playlist = { @@ -55,8 +53,8 @@ module.exports = { }; // Temp fix for broken v2 playlist. - player.href = src.replace(/\/embed\/playlist\-v2\//, '/embed/playlist/'); - } else if (/episode|show/.test(src)) { + player.href = iframe.src.replace(/\/embed\/playlist\-v2\//, '/embed/playlist/'); + } else if (/episode|show/.test(iframe.src)) { player.rel.push(CONFIG.R.audio); player.height = iframe.height || 232; } else { diff --git a/plugins/domains/strawpoll.me.js b/plugins/domains/strawpoll.me.js index 20066396f..d270a4ca5 100644 --- a/plugins/domains/strawpoll.me.js +++ b/plugins/domains/strawpoll.me.js @@ -1,7 +1,6 @@ -var $ = require('cheerio'); -var entities = require('entities'); +import * as entities from 'entities'; -module.exports = { +export default { re: /^https?:\/\/(?:www\.)?strawpoll\.me\/([0-9]+)$/i, @@ -10,7 +9,7 @@ module.exports = { getLink: function(urlMatch, cheerio) { var embed = entities.decodeHTML(cheerio('#embed-field-' + urlMatch[1]).attr('value')); - var iframe = $('<div>').html(embed).children('iframe'); + var iframe = cheerio('<div>').html(embed).children('iframe'); var height = parseInt(iframe.css('height').replace(/px$/, ''), 10) || 300; var width = parseInt(iframe.css('width').replace(/px$/, ''), 10) || 690; @@ -26,7 +25,7 @@ module.exports = { }, tests: [ - "http://www.strawpoll.me/1696", + "https://www.strawpoll.me/1696", "https://strawpoll.me/136" ] }; diff --git a/plugins/domains/sverigesradio.se.js b/plugins/domains/sverigesradio.se.js index 4fb71b863..82b15e34b 100644 --- a/plugins/domains/sverigesradio.se.js +++ b/plugins/domains/sverigesradio.se.js @@ -1,4 +1,4 @@ -module.exports = { +export default { // This is for articles only - players are covered by oembed auto-discovery re: [ @@ -14,8 +14,6 @@ module.exports = { request({ uri: "http://sverigesradio.se/sida/playerajax/getaudiourl?id=" + urlMatch[1] + "&type=publication&quality=medium&format=iis", - limit: 1, - timeout: 1000, prepareResult: function(error, response, body, cb) { // 404 means article is not embeddable diff --git a/plugins/domains/ted.com/ted.com.js b/plugins/domains/ted.com/ted.com.js index 3d88243c7..a36b6dab7 100644 --- a/plugins/domains/ted.com/ted.com.js +++ b/plugins/domains/ted.com/ted.com.js @@ -1,6 +1,6 @@ -const URL = require('url'); +import * as URL from 'url'; -module.exports = { +export default { re: /^https?:\/\/(?:www\.)?ted\.com\/talks\//i, diff --git a/plugins/domains/ted.com/ted.playlists.js b/plugins/domains/ted.com/ted.playlists.js index daed00cf6..d3a0769be 100644 --- a/plugins/domains/ted.com/ted.playlists.js +++ b/plugins/domains/ted.com/ted.playlists.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/(?:www\.)?ted\.com\/playlists\//i, diff --git a/plugins/domains/test.com/test1.js b/plugins/domains/test.com/test1.js index d82cda261..01ef363d1 100644 --- a/plugins/domains/test.com/test1.js +++ b/plugins/domains/test.com/test1.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: 'test_data1', diff --git a/plugins/domains/test.com/test2.js b/plugins/domains/test.com/test2.js index 519bfb8cf..33fd1557c 100644 --- a/plugins/domains/test.com/test2.js +++ b/plugins/domains/test.com/test2.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: ['test_data2', 'test_plugin'], diff --git a/plugins/domains/test.com/test3.js b/plugins/domains/test.com/test3.js index 45843d327..7e0d746cc 100644 --- a/plugins/domains/test.com/test3.js +++ b/plugins/domains/test.com/test3.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(test_data2) { return { diff --git a/plugins/domains/theatlas.com/theatlas.com.js b/plugins/domains/theatlas.com/theatlas.com.js index db2f7ff29..04dd29855 100644 --- a/plugins/domains/theatlas.com/theatlas.com.js +++ b/plugins/domains/theatlas.com/theatlas.com.js @@ -1,6 +1,6 @@ -var utils = require('../../../lib/utils'); +import * as utils from '../../../lib/utils.js'; -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.)?theatlas\.com\/charts\/([a-zA-Z0-9]+)/i diff --git a/plugins/domains/theguardian.com/guardian.video.js b/plugins/domains/theguardian.com/guardian.video.js index 9e2ebcafd..352d42157 100644 --- a/plugins/domains/theguardian.com/guardian.video.js +++ b/plugins/domains/theguardian.com/guardian.video.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /https?:\/\/www\.theguardian\.com\/[\w-]+\/video\/\d+\/\w+\/\d+\/[\w-]+/i, diff --git a/plugins/domains/theta360.com/theta360.com.js b/plugins/domains/theta360.com/theta360.com.js index 7f6c185b4..0cdffe9c3 100644 --- a/plugins/domains/theta360.com/theta360.com.js +++ b/plugins/domains/theta360.com/theta360.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /https:\/\/theta360\.com\/(?:spheres|s)(?:\/\w+)?\/[\w-]+/i, diff --git a/plugins/domains/tmz.com.js b/plugins/domains/tmz.com.js index 345a1040d..fcca586c1 100644 --- a/plugins/domains/tmz.com.js +++ b/plugins/domains/tmz.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.tmz\.com\/videos\/([a-zA-Z0-9_]+)/, diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js index 0cbdd25d3..a3a44edf1 100644 --- a/plugins/domains/tumblr.com/tumblr.api.js +++ b/plugins/domains/tumblr.com/tumblr.api.js @@ -1,7 +1,7 @@ -var $ = require('cheerio'); -var _ = require('underscore'); +import cheerio from 'cheerio'; -module.exports = { + +export default { re: [ /^https?:\/\/([a-z0-9-]+\.tumblr\.com)\/(post|image)\/(\d+)(?:\/[a-z0-9-]+)?/i, @@ -12,7 +12,7 @@ module.exports = { getMeta: function(tumblr_post) { - var caption = tumblr_post.caption ? $('<div>').html(tumblr_post.caption).text() : ""; + var caption = tumblr_post.caption ? cheerio('<div>').html(tumblr_post.caption).text() : ""; if (caption && caption.length > 160) { caption = caption.split(/[.,!?]/)[0]; } @@ -23,11 +23,11 @@ module.exports = { author: tumblr_post.blog_name, author_url: 'https://' + tumblr_post.blog_name + '.tumblr.com', canonical: tumblr_post.permalink_url || tumblr_post.post_url, - tags: _.unique([].concat(tumblr_post.tags, tumblr_post.featured_in_tag || [])).join(', '), + tags: tumblr_post.tags && tumblr_post.tags.join(', '), shortlink: tumblr_post.short_url, date: tumblr_post.date, duration: tumblr_post.duration, - description: tumblr_post.body && /<p/.test(tumblr_post.body) ? $('<div>').html(tumblr_post.body).find('p').first().text() : null + description: tumblr_post.body && /<p/.test(tumblr_post.body) ? cheerio('<div>').html(tumblr_post.body).find('p').first().text() : null }; }, @@ -75,12 +75,10 @@ module.exports = { id: urlMatch[3] }, json: true, - limit: 1, - timeout: 1000, prepareResult: function (error, response, body, cb) { if (error || body.errors) { - return cb(error || 'There was a Tumblr API error error'); + return cb(error || body.errors || 'There was a Tumblr API error'); } if (!body.meta) { diff --git a/plugins/domains/tumblr.com/tumblr.oembed.js b/plugins/domains/tumblr.com/tumblr.oembed.js index 08c85c763..640647f80 100644 --- a/plugins/domains/tumblr.com/tumblr.oembed.js +++ b/plugins/domains/tumblr.com/tumblr.oembed.js @@ -1,8 +1,10 @@ /* Tumblr embed codes are broken as of Feb 13, 2020 */ -module.exports = { +import tumblr_api from './tumblr.api.js'; - re: require('./tumblr.api').re, +export default { + + re: tumblr_api.re, getLink: function(tumblr_post, oembed, options) { diff --git a/plugins/domains/tumblr.com/tumblr.photo.js b/plugins/domains/tumblr.com/tumblr.photo.js index d4eefd06d..914f6ec7b 100644 --- a/plugins/domains/tumblr.com/tumblr.photo.js +++ b/plugins/domains/tumblr.com/tumblr.photo.js @@ -1,9 +1,12 @@ -var _ = require('underscore'); -var $ = require('cheerio'); +import * as _ from 'underscore'; +import cheerio from 'cheerio'; -module.exports = { - re: require('./tumblr.api').re, +import tumblr_api from './tumblr.api.js'; + +export default { + + re: tumblr_api.re, getLinks: function(tumblr_post) { @@ -27,7 +30,7 @@ module.exports = { tumblr_post.photos.forEach(function(photo) { var title = photo.caption || tumblr_post.caption; - title = $('<div>').html(title).text(); + title = cheerio('<div>').html(title).text(); if (title && title.length > 160) { title = title.split(/[.,!?]/)[0]; } diff --git a/plugins/domains/tumblr.com/tumblr.text.js b/plugins/domains/tumblr.com/tumblr.text.js index aae3a86ef..dc3d59222 100644 --- a/plugins/domains/tumblr.com/tumblr.text.js +++ b/plugins/domains/tumblr.com/tumblr.text.js @@ -1,8 +1,10 @@ -var $ = require('cheerio'); +import cheerio from 'cheerio'; -module.exports = { +import tumblr_api from './tumblr.api.js'; - re: require('./tumblr.api').re, +export default { + + re: tumblr_api.re, getMeta: function (tumblr_post) { if (tumblr_post.type == "text") { @@ -18,7 +20,7 @@ module.exports = { return; } - var $post = $('<div>').html(tumblr_post.body); + var $post = cheerio('<div>').html(tumblr_post.body); var $image = $post.find('img').first(); // Could be more than 1 image, true. But the response time will be unacceptable as post-processing will check all image sizes. if ($image ) { diff --git a/plugins/domains/tumblr.com/tumblr.video.js b/plugins/domains/tumblr.com/tumblr.video.js index 8fca974f5..7a3f02ff6 100644 --- a/plugins/domains/tumblr.com/tumblr.video.js +++ b/plugins/domains/tumblr.com/tumblr.video.js @@ -1,8 +1,10 @@ -const $ = require('cheerio'); +import cheerio from 'cheerio'; -module.exports = { +import tumblr_api from './tumblr.api.js'; - re: require('./tumblr.api').re, +export default { + + re: tumblr_api.re, getLink: function(tumblr_post) { @@ -24,7 +26,7 @@ module.exports = { var p = tumblr_post.player instanceof Array ? tumblr_post.player[0] : tumblr_post.player; - var $c = $('<div>').append(p.embed_code); + var $c = cheerio('<div>').append(p.embed_code); var $iframe = $c.find('iframe'); if ($iframe.length) { diff --git a/plugins/domains/twitch.tv/clips.twitch.tv.js b/plugins/domains/twitch.tv/clips.twitch.tv.js index e76ced53b..21ab7d97c 100644 --- a/plugins/domains/twitch.tv/clips.twitch.tv.js +++ b/plugins/domains/twitch.tv/clips.twitch.tv.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/clips.twitch\.tv\/([^\?\/]+)(?:\?[^\?]+)?$/i, diff --git a/plugins/domains/twitch.tv/twitch-live-fallback.js b/plugins/domains/twitch.tv/twitch-live-fallback.js index 5290ef45f..5100ce205 100644 --- a/plugins/domains/twitch.tv/twitch-live-fallback.js +++ b/plugins/domains/twitch.tv/twitch-live-fallback.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.|go\.)?twitch\.tv\/([a-zA-Z0-9_]+)(?:\?parent=.*)?$/i, diff --git a/plugins/domains/twitch.tv/twitch.tv.js b/plugins/domains/twitch.tv/twitch.tv.js index e2903ad03..19d434855 100644 --- a/plugins/domains/twitch.tv/twitch.tv.js +++ b/plugins/domains/twitch.tv/twitch.tv.js @@ -1,6 +1,6 @@ -const URL = require('url'); +import * as URL from 'url'; -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.|go\.)?twitch\.tv\/[a-zA-Z0-9_]+\/v\/(\d+)/i, diff --git a/plugins/domains/twitter.com/twitter.og.js b/plugins/domains/twitter.com/twitter.og.js index 990e7732d..bdd394715 100644 --- a/plugins/domains/twitter.com/twitter.og.js +++ b/plugins/domains/twitter.com/twitter.og.js @@ -1,6 +1,8 @@ -module.exports = { +import twitter_status from './twitter.status.js'; - re: require('./twitter.status').re, +export default { + + re: twitter_status.re, provides: ['twitter_og'], diff --git a/plugins/domains/twitter.com/twitter.status.js b/plugins/domains/twitter.com/twitter.status.js index acb3ffd20..b5a7b8f50 100644 --- a/plugins/domains/twitter.com/twitter.status.js +++ b/plugins/domains/twitter.com/twitter.status.js @@ -1,182 +1,81 @@ -var async = require('async'); -var cache = require('../../../lib/cache'); -var sysUtils = require('../../../logging'); -var _ = require('underscore'); -var entities = require('entities'); +import log from '../../../logging.js'; +import * as entities from 'entities'; -module.exports = { +export default { - re: [ - /^https?:\/\/twitter\.com\/(?:\w+)\/status(?:es)?\/(\d+)/i - ], + re: /^https?:\/\/twitter\.com\/(?:\w+)\/status(?:es)?\/(\d+)/i, provides: ['twitter_oembed', 'twitter_og', '__allowTwitterOg'], mixins: ['domain-icon'], getData: function(urlMatch, request, options, cb) { - var id = urlMatch[1]; var c = options.getProviderOptions("twitter") || options.getProviderOptions("twitter.status"); if (c.disabled) { - return cb('Twitter API Disabled'); + return cb('Twitter API disabled'); } - var oauth = c.consumer_key - ? { - consumer_key: c.consumer_key, - consumer_secret: c.consumer_secret, - token: c.access_token, - token_secret: c.access_token_secret - } : false; - - var blockExpireIn = 0; - var block_key = 'twbl:' + c.consumer_key; - - async.waterfall([ + request({ + url: "https://publish.twitter.com/oembed", + qs: { + hide_media: c.hide_media, + hide_thread: true, // c.hide_thread - now handled in getLinks. This is the only reliable way to detect if a tweet has the thread + omit_script: c.omit_script, + url: urlMatch[0] + }, + json: true, + cache_key: 'twitter:oembed:' + urlMatch[1], + prepareResult: function(error, response, oembed, cb) { - function(cb) { + if (error) { + return cb(error); + } - if (oauth) { - cache.get(block_key, cb); - } else { - cb(null, null); + if (response.fromRequestCache) { + log(' -- Twitter API cache used.'); } - }, - function(expireIn, cb) { + if (response.statusCode === 404) { + return cb({ + responseStatusCode: 404, + message: 'The tweet is no longer available.' + }) + + } else if (response.statusCode === 403) { + return cb({ + responseStatusCode: 404, + message: 'It looks this Twitter account has been suspended.' + }) - if (expireIn) { - var now = Math.round(new Date().getTime() / 1000); - if (expireIn > now) { - blockExpireIn = expireIn - now; - } + } else if (response.statusCode !== 200) { + return cb('Non-200 response from Twitter API (statuses/oembed.json: ' + response.statusCode); } - var usePublicApi = !oauth || blockExpireIn > 0; + if (typeof oembed !== 'object') { + return cb('Object expected in Twitter API (statuses/oembed.json), got: ' + oembed); + } - var apiUrl; + oembed.title = oembed.author_name + ' on Twitter'; + oembed["min-width"] = c["min-width"]; + oembed["max-width"] = c["max-width"]; - var qs = { - hide_media: c.hide_media, - hide_thread: true, // c.hide_thread - now handled in getLinks. This is the only reliable way to detect if a tweet has the thread - omit_script: c.omit_script + var result = { + twitter_oembed: oembed }; - if (usePublicApi) { - apiUrl = "https://publish.twitter.com/oembed"; - qs.url = urlMatch[0]; + if (/pic\.twitter\.com/i.test(oembed.html)) { + result.__allowTwitterOg = true; + options.followHTTPRedirect = true; // avoid core's re-directs. Use HTTP request redirects instead + options.exposeStatusCode = true; } else { - apiUrl = "https://api.twitter.com/1.1/statuses/oembed.json"; - qs.id = id; + result.twitter_og = false; } - request(_.extend({ - url: apiUrl, - qs: qs, - json: true, - cache_key: 'twitter:oembed:' + id, - prepareResult: function(error, response, data, cb) { - - if (error) { - return cb(error); - } - - if (response.fromRequestCache) { - if (blockExpireIn > 0) { - sysUtils.log(' -- Twitter API limit reached (' + blockExpireIn + ' seconds left), but cache used.'); - } else { - sysUtils.log(' -- Twitter API cache used.'); - } - } - - // Do not block 1.1 api if data from cache. - if (oauth && !response.fromRequestCache) { - - var remaining = parseInt(response.headers['x-rate-limit-remaining']); - - if (response.statusCode === 429 || remaining <= 7) { - var now = Math.round(new Date().getTime() / 1000); - var limitResetAt = parseInt(response.headers['x-rate-limit-reset']); - var ttl = limitResetAt - now; - - // Do not allow ttl 0. - // 5 seconds - to cover possible time difference with twitter. - if (ttl < 5) { - ttl = 5; - } - - // Block maximum for 15 minutes. - if (ttl > 15*60) { - ttl = 15*60 - } - - if (response.statusCode === 429) { - sysUtils.log(' -- Twitter API limit reached by status code 429. Disabling for ' + ttl + ' seconds.'); - } else { - sysUtils.log(' -- Twitter API limit warning, remaining calls: ' + remaining + '. Disabling for ' + ttl + ' seconds.'); - } - - // Store expire date as value to be sure it past. - var expireIn = now + ttl; - - cache.set(block_key, expireIn, {ttl: ttl}); - } - } - - if (response.statusCode === 404) { - return cb({ - responseStatusCode: 404, - message: 'The tweet is no longer available.' - }) - } else if (response.statusCode === 403) { - return cb({ - responseStatusCode: 404, - message: 'It looks this Twitter account has been suspended.' - }) - - } else if (response.statusCode !== 200) { - return cb('Non-200 response from Twitter API (statuses/oembed.json: ' + response.statusCode); - } - - if (typeof data !== 'object') { - return cb('Object expected in Twitter API (statuses/oembed.json), got: ' + data); - } - - - cb(error, data); - } - }, usePublicApi ? null : {oauth: oauth}), cb); // add oauth if 1.1, else skip it - - } - - ], function(error, oembed) { - - - if (error) { - return cb(error); - } - - oembed.title = oembed.author_name + ' on Twitter'; - - oembed["min-width"] = c["min-width"]; - oembed["max-width"] = c["max-width"]; - - var result = { - twitter_oembed: oembed - }; - - if (/pic\.twitter\.com/i.test(oembed.html)) { - result.__allowTwitterOg = true; - options.followHTTPRedirect = true; // avoid core's re-directs. Use HTTP request redirects instead - options.exposeStatusCode = true; - } else { - result.twitter_og = false; + return cb(error, result); } - - cb(null, result); - }); + }, cb); }, getMeta: function(twitter_oembed) { diff --git a/plugins/domains/twitter.com/twitter.throttle.js b/plugins/domains/twitter.com/twitter.throttle.js index 2e5ee6ca2..70d49b683 100644 --- a/plugins/domains/twitter.com/twitter.throttle.js +++ b/plugins/domains/twitter.com/twitter.throttle.js @@ -1,6 +1,9 @@ -module.exports = { +import twitter_timelines from './twitter.timelines.js'; +import twitter_status from './twitter.status.js'; - re: require('./twitter.timelines').re.concat(require('./twitter.status').re), +export default { + + re: twitter_timelines.re.concat(twitter_status.re), provides: ["meta"], diff --git a/plugins/domains/twitter.com/twitter.timelines.js b/plugins/domains/twitter.com/twitter.timelines.js index 2ed3fb7fd..561e8f41e 100644 --- a/plugins/domains/twitter.com/twitter.timelines.js +++ b/plugins/domains/twitter.com/twitter.timelines.js @@ -1,4 +1,4 @@ -module.exports = { +export default { // Embedded Like, Collection, and Moment Timelines are now retired. // https://twittercommunity.com/t/removing-support-for-embedded-like-collection-and-moment-timelines/150313 diff --git a/plugins/domains/usatoday-allowEmbedURL.js b/plugins/domains/usatoday-allowEmbedURL.js index 9692ecfcb..a3528d11e 100644 --- a/plugins/domains/usatoday-allowEmbedURL.js +++ b/plugins/domains/usatoday-allowEmbedURL.js @@ -1,6 +1,8 @@ -module.exports = { +import usatoday_com from './usatoday.com.js'; - re: require('./usatoday.com').re, +export default { + + re: usatoday_com.re, provides: '__allowEmbedURL', diff --git a/plugins/domains/usatoday.com.js b/plugins/domains/usatoday.com.js index d6caf9a6e..605f52ba5 100644 --- a/plugins/domains/usatoday.com.js +++ b/plugins/domains/usatoday.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /^https?:\/\/www\.([a-z\-]+)\.com\/media\/cinematic\/video\/(\d{7,12})\/[a-zA-Z0-9\-\_:\.]+\/?(?:[^\/]+)?$/i, diff --git a/plugins/domains/v.qq.com.js b/plugins/domains/v.qq.com.js index f56561d83..920b079e3 100644 --- a/plugins/domains/v.qq.com.js +++ b/plugins/domains/v.qq.com.js @@ -1,4 +1,4 @@ -var _ = require('underscore'); +import * as _ from 'underscore'; var res = [ /^https?:\/\/v\.qq\.com\/page\/\w\/\w\/\w\/(\w+)\.html$/i, @@ -8,7 +8,7 @@ var res = [ /^https?:\/\/v\.qq\.com\/\w\/cover\/\w+\/(\w+)\.html/i ]; -module.exports = { +export default { re: res, diff --git a/plugins/domains/vbox7.com.js b/plugins/domains/vbox7.com.js index 007b67346..3e1ed0334 100644 --- a/plugins/domains/vbox7.com.js +++ b/plugins/domains/vbox7.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/(?:www\.)?vbox7\.com\/play:(\w+)/i diff --git a/plugins/domains/vimeo-moogaloop.js b/plugins/domains/vimeo-moogaloop.js index 2d43c1f46..edcb01730 100644 --- a/plugins/domains/vimeo-moogaloop.js +++ b/plugins/domains/vimeo-moogaloop.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/vimeo\.com\/moogaloop\.swf\?clip_id=(\d+)/i, diff --git a/plugins/domains/vimeo.com.js b/plugins/domains/vimeo.com.js index cad8493a5..47a0320ac 100644 --- a/plugins/domains/vimeo.com.js +++ b/plugins/domains/vimeo.com.js @@ -1,6 +1,6 @@ -const querystring = require('querystring'); +import * as querystring from 'querystring'; -module.exports = { +export default { re: [ /^https:\/\/vimeo\.com(?:\/channels?\/\w+)?\/\d+/i, // Includes private reviews like /video/123/ABC. diff --git a/plugins/domains/washingtonpost.com.js b/plugins/domains/washingtonpost.com.js index 2ebe6450d..1042f07fc 100644 --- a/plugins/domains/washingtonpost.com.js +++ b/plugins/domains/washingtonpost.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https:\/\/www\.washingtonpost\.com\/(?:video|posttv)\/\w+\/[a-z0-9-]+\/\d+\/\d+\/\d+\/([a-z0-9-]+)_video\.html/i, diff --git a/plugins/domains/widgetic.com.js b/plugins/domains/widgetic.com.js index 9af8e8c5f..78b1b8167 100644 --- a/plugins/domains/widgetic.com.js +++ b/plugins/domains/widgetic.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: [ /https?:\/\/widgetic\.com\/widgets\//i, diff --git a/plugins/domains/wistia.com.js b/plugins/domains/wistia.com.js index 3bf63667b..796489b46 100644 --- a/plugins/domains/wistia.com.js +++ b/plugins/domains/wistia.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/([a-zA-Z0-9]+).wistia\.com\/medias\/([_a-zA-Z0-9]+)/i, diff --git a/plugins/domains/xkcd.com.js b/plugins/domains/xkcd.com.js index a65114092..76ea42fe0 100644 --- a/plugins/domains/xkcd.com.js +++ b/plugins/domains/xkcd.com.js @@ -1,6 +1,6 @@ -const decodeHTML5 = require('entities').decodeHTML5; +import { decodeHTML5 } from 'entities'; -module.exports = { +export default { re: /^https?:\/\/(?:www.)?xkcd\.com\/\d+/i, diff --git a/plugins/domains/youku.com.js b/plugins/domains/youku.com.js index ff3c36a45..96dd4c3cb 100644 --- a/plugins/domains/youku.com.js +++ b/plugins/domains/youku.com.js @@ -1,4 +1,4 @@ -module.exports = { +export default { //http://v.youku.com/v_show/id_XNDkwNjg2NzQw.html?f=18736842 re: [ diff --git a/plugins/domains/youtube.com/youtube.channel.js b/plugins/domains/youtube.com/youtube.channel.js index c72cfee42..8055eed56 100644 --- a/plugins/domains/youtube.com/youtube.channel.js +++ b/plugins/domains/youtube.com/youtube.channel.js @@ -1,6 +1,6 @@ -const sysUtils = require('../../../logging') +import log from '../../../logging.js' -module.exports = { +export default { re: [ /^https?:\/\/(?:www\.)?youtube\.com\/(channel)\/([a-zA-Z0-9_-]+)/i, @@ -90,7 +90,7 @@ module.exports = { * as if no channel found, but it actually exists. * Ex.: https://www.youtube.com/c/Figmadesign */ - sysUtils.log('YoutTube channel fallback for ' + url , data); + log('YoutTube channel fallback for ' + url , data); cb({ message: 'YouTube channel not found via data API...' // But no response code. Let's fallback to default parsers diff --git a/plugins/domains/youtube.com/youtube.playlist.js b/plugins/domains/youtube.com/youtube.playlist.js index 2e6cdde9d..d1bf1c0ec 100644 --- a/plugins/domains/youtube.com/youtube.playlist.js +++ b/plugins/domains/youtube.com/youtube.playlist.js @@ -1,6 +1,6 @@ -const querystring = require('querystring'); +import * as querystring from 'querystring'; -module.exports = { +export default { re: [ /^https?:\/\/www\.youtube\.com\/playlist\?(?:[=\-_a-zA-Z0-9&]+)?list=([\-_a-zA-Z0-9]+)/i diff --git a/plugins/domains/youtube.com/youtube.shared.js b/plugins/domains/youtube.com/youtube.shared.js index 86dd1d5a2..3b8778862 100644 --- a/plugins/domains/youtube.com/youtube.shared.js +++ b/plugins/domains/youtube.com/youtube.shared.js @@ -1,4 +1,4 @@ -module.exports = { +export default { re: /^https?:\/\/www\.youtube\.com\/shared\/?\?ci=([a-zA-Z0-9_-]+)/i, diff --git a/plugins/domains/youtube.com/youtube.video.js b/plugins/domains/youtube.com/youtube.video.js index d2b266215..26d8d262e 100644 --- a/plugins/domains/youtube.com/youtube.video.js +++ b/plugins/domains/youtube.com/youtube.video.js @@ -1,9 +1,10 @@ -const cheerio = require('cheerio'); -const querystring = require('querystring'); -const _ = require('underscore'); -const sysUtils = require('../../../logging') +import cheerio from 'cheerio'; -module.exports = { +import * as querystring from 'querystring'; +import * as _ from 'underscore'; +import log from '../../../logging.js' + +export default { re: [ /^https?:\/\/(?:www\.)?youtube\.com\/(?:tv#\/)?watch\/?\?(?:[^&]+&)*v=([a-zA-Z0-9_-]+)/i, @@ -129,7 +130,7 @@ module.exports = { } else if (data.items && data.items.length == 0 || data.error && data.error.code == 404) { cb({responseStatusCode: 404}); } else { - sysUtils.log('YoutTube fallback for ' + urlMatch[1], data); + log('YoutTube fallback for ' + urlMatch[1], data); cb(null); // silence error for fallback to generic providers. data.error.code == 429 - too many requests; 400 - probably API key is invalid } } diff --git a/plugins/links/article/article.js b/plugins/links/article/article.js index 72821e6d0..d603f9154 100644 --- a/plugins/links/article/article.js +++ b/plugins/links/article/article.js @@ -1,6 +1,6 @@ -var utils = require('../../../lib/utils'); +import * as utils from '../../../lib/utils.js'; -module.exports = { +export default { getData: function(readability, meta, __is_general_article) { diff --git a/plugins/links/article/check-article.js b/plugins/links/article/check-article.js index 10e77f3d8..11185238e 100644 --- a/plugins/links/article/check-article.js +++ b/plugins/links/article/check-article.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: '__is_general_article', diff --git a/plugins/links/canonical-redirect.js b/plugins/links/canonical-redirect.js index 679260b11..36f8fb4c5 100644 --- a/plugins/links/canonical-redirect.js +++ b/plugins/links/canonical-redirect.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(url, meta, options, cb) { diff --git a/plugins/links/embedURL/checkEmbedURL.js b/plugins/links/embedURL/checkEmbedURL.js index 350793ecc..beb89965e 100644 --- a/plugins/links/embedURL/checkEmbedURL.js +++ b/plugins/links/embedURL/checkEmbedURL.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: '__allowEmbedURL', diff --git a/plugins/links/embedURL/embedURL.js b/plugins/links/embedURL/embedURL.js index 48c747dfc..66a03de4e 100644 --- a/plugins/links/embedURL/embedURL.js +++ b/plugins/links/embedURL/embedURL.js @@ -1,7 +1,7 @@ -const decodeHTML5 = require('entities').decodeHTML5; -const utils = require('../../../lib/utils'); +import { decodeHTML5 } from 'entities'; +import * as utils from '../../../lib/utils.js'; -module.exports = { +export default { provides: 'schemaVideoObject', @@ -12,10 +12,10 @@ module.exports = { if ($script.length === 1) { try { - var json = utils.parseJSONSource($script.text()); + var json = utils.parseJSONSource($script.html()); if (json['@type']) { - ld = {}; + var ld = {}; ld[json['@type'].toLowerCase()] = json; if (__allowEmbedURL !== 'skip_ld') { diff --git a/plugins/links/embedURL/ld-video.js b/plugins/links/embedURL/ld-video.js index a97fd2d69..cc0e978a7 100644 --- a/plugins/links/embedURL/ld-video.js +++ b/plugins/links/embedURL/ld-video.js @@ -1,6 +1,7 @@ -const cheerio = require('cheerio'); +import cheerio from 'cheerio'; -module.exports = { + +export default { provides: [ 'schemaVideoObject', diff --git a/plugins/links/favicon.js b/plugins/links/favicon.js index 5817d3250..367d95aac 100644 --- a/plugins/links/favicon.js +++ b/plugins/links/favicon.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(meta) { diff --git a/plugins/links/google-docs-viewer.js b/plugins/links/google-docs-viewer.js index 17ea34970..e6215fcfb 100644 --- a/plugins/links/google-docs-viewer.js +++ b/plugins/links/google-docs-viewer.js @@ -1,6 +1,6 @@ -const utils = require('../../lib/utils'); +import * as utils from '../../lib/utils.js'; -module.exports = { +export default { getLink: function(url, __nonHtmlContentData, options) { diff --git a/plugins/links/hosted/23video-hosted.js b/plugins/links/hosted/23video-hosted.js index 8a163a59f..9ae4bb470 100644 --- a/plugins/links/hosted/23video-hosted.js +++ b/plugins/links/hosted/23video-hosted.js @@ -1,47 +1,18 @@ -var cheerio = require('cheerio'); +export default { -module.exports = { + provides: ["oembed_domain"], // linked to oembed, so won't run for all URLs - getLink: function(meta, oembed, whitelistRecord) { + getData: function(meta, oembed, iframe, whitelistRecord) { if (whitelistRecord.isDefault - && oembed.type == 'video' && oembed.width && oembed.height - && ((meta.em && meta.em.schema == '23video') || oembed.provider_name == 'TwentyThree')) { + && oembed.type == 'video' + && /\/v\.ihtml(?:\/player\.html)?/i.test(iframe.src) + && ((meta.em && meta.em.schema == '23video') || /^twentythree$/i.test(oembed.provider_name))) { - var players = []; - - if (meta.video_src - && /https?:\/\/[^.]+\.23video\.com\/\d+\/\d+\/\w+\/[^\?]+\.mp4/i.test(meta.video_src)) { - - players.push({ - href: meta.video_src, - type: CONFIG.T.video_mp4, - rel: [CONFIG.R.player, CONFIG.R.html5], - 'aspect-ratio': oembed.width / oembed.height - }); + return { + oembed_domain: "23video.com" } - - var $container = cheerio('<div>'); - try { - $container.html(oembed.html); - } catch (ex) {} - - var $iframe = $container.find('iframe'); - - if ($iframe.length == 1 && /\/v\.ihtml(?:\/player\.html)?/i.test($iframe.attr('src'))) { - - players.push({ - href: $iframe.attr('src'), - type: CONFIG.T.text_html, - rel: [CONFIG.R.player, CONFIG.R.html5], - 'aspect-ratio': oembed.width / oembed.height, - autoplay: 'autoPlay=1' - }); - - } - - return players; } }, @@ -51,7 +22,7 @@ module.exports = { "https://video.twentythree.net/intro-to-twentythrees-player-builder", "http://videos.theconference.se/paul-adams-solving-real-world-problems", "http://www.fftv.no/skipatruljen-s3e3-voss-resort", - "https://videos.23video.com/novo-nordisk", + // "https://videos.23video.com/novo-nordisk", "http://video.nextconf.eu/video/1880845/data-without-limits", "http://stream.umbraco.org/v.ihtml?source=share&photo%5fid=11665495&autoPlay=0" ] diff --git a/plugins/links/hosted/brightcove-allow-in-page.js b/plugins/links/hosted/brightcove-allow-in-page.js index 43a4b0b18..97c1b13a1 100644 --- a/plugins/links/hosted/brightcove-allow-in-page.js +++ b/plugins/links/hosted/brightcove-allow-in-page.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: '__allowBrightcoveInPage', diff --git a/plugins/links/hosted/brightcove-hosted.js b/plugins/links/hosted/brightcove-hosted.js index c62629355..11cca8688 100644 --- a/plugins/links/hosted/brightcove-hosted.js +++ b/plugins/links/hosted/brightcove-hosted.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: '__promoUri', diff --git a/plugins/links/hosted/brightcove-in-page-promo.js b/plugins/links/hosted/brightcove-in-page-promo.js index 2165ff3f2..d159ea295 100644 --- a/plugins/links/hosted/brightcove-in-page-promo.js +++ b/plugins/links/hosted/brightcove-in-page-promo.js @@ -3,7 +3,7 @@ * http://docs.brightcove.com/en/perform/brightcove-player/guides/embed-in-page.html */ -module.exports = { +export default { provides: '__promoUri', diff --git a/plugins/links/hosted/cloudapp-hosted.js b/plugins/links/hosted/cloudapp-hosted.js index f79d77485..8e16540ca 100644 --- a/plugins/links/hosted/cloudapp-hosted.js +++ b/plugins/links/hosted/cloudapp-hosted.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(twitter) { diff --git a/plugins/links/hosted/cnevids-hosted.js b/plugins/links/hosted/cnevids-hosted.js index 86ae5f9e5..91fda2f30 100644 --- a/plugins/links/hosted/cnevids-hosted.js +++ b/plugins/links/hosted/cnevids-hosted.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getData: function(video_src) { diff --git a/plugins/links/hosted/hosted-via-canonical.js b/plugins/links/hosted/hosted-via-canonical.js index 343dfb785..4f3385190 100644 --- a/plugins/links/hosted/hosted-via-canonical.js +++ b/plugins/links/hosted/hosted-via-canonical.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(url, meta, options, cb) { diff --git a/plugins/links/hosted/known-players.js b/plugins/links/hosted/known-players.js index b88991a7b..1b1a1fdb5 100644 --- a/plugins/links/hosted/known-players.js +++ b/plugins/links/hosted/known-players.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: '__promoUri', diff --git a/plugins/links/hosted/libsyn-hosted.js b/plugins/links/hosted/libsyn-hosted.js index b65a4cfb4..cb86ac0fe 100644 --- a/plugins/links/hosted/libsyn-hosted.js +++ b/plugins/links/hosted/libsyn-hosted.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getData: function(url, video_src, twitter) { diff --git a/plugins/links/hosted/ooyala.js b/plugins/links/hosted/ooyala.js index 53f747ec1..5626e229d 100644 --- a/plugins/links/hosted/ooyala.js +++ b/plugins/links/hosted/ooyala.js @@ -1,6 +1,6 @@ -var URL = require('url'); +import * as URL from 'url'; -module.exports = { +export default { provides: '__ooyalaPlayer', @@ -38,7 +38,6 @@ module.exports = { method: 'HEAD', headers: { 'Accept-Encoding': 'gzip, deflate, sdch', // this is required for head request to return content_length - 'Connection': 'close' }, prepareResult: function(error, response, body, cb) { diff --git a/plugins/links/hosted/podbean-hosted.js b/plugins/links/hosted/podbean-hosted.js index e17ec1b11..23ff977ce 100644 --- a/plugins/links/hosted/podbean-hosted.js +++ b/plugins/links/hosted/podbean-hosted.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getData: function(video_src) { diff --git a/plugins/links/hosted/promo-link.js b/plugins/links/hosted/promo-link.js index e32041623..627cd62be 100644 --- a/plugins/links/hosted/promo-link.js +++ b/plugins/links/hosted/promo-link.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: '__promoUri', diff --git a/plugins/links/hosted/promo-oembed.js b/plugins/links/hosted/promo-oembed.js index 034683eb8..5ffbbf58c 100644 --- a/plugins/links/hosted/promo-oembed.js +++ b/plugins/links/hosted/promo-oembed.js @@ -1,6 +1,6 @@ -var urlLib = require('url'); +import * as urlLib from 'url'; -module.exports = { +export default { provides: '__promoUri', diff --git a/plugins/links/hosted/promo.js b/plugins/links/hosted/promo.js index 979f4d028..ee0910de8 100644 --- a/plugins/links/hosted/promo.js +++ b/plugins/links/hosted/promo.js @@ -1,11 +1,10 @@ -var core = require('../../../lib/core'); -var _ = require('underscore'); +import * as _ from 'underscore'; -module.exports = { +export default { provides: 'self', - getData: function(url, __promoUri, options, cb) { + getData: function(url, __promoUri, iframelyRun, options, cb) { // __promoUri may be not string if no rel=promo need to be added // see theplatform plugin for example @@ -24,7 +23,7 @@ module.exports = { delete options2.promoUri; delete options2.jar; - core.run(promoUri, options2, function(error, data) { + iframelyRun(promoUri, options2, function(error, data) { var wrappedError = null; diff --git a/plugins/links/hosted/tout-hosted.js b/plugins/links/hosted/tout-hosted.js index 3104815e3..3a1df1cb9 100644 --- a/plugins/links/hosted/tout-hosted.js +++ b/plugins/links/hosted/tout-hosted.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(twitter, whitelistRecord) { diff --git a/plugins/links/hosted/video_src-hosted.js b/plugins/links/hosted/video_src-hosted.js index fb40678ea..d8feea74c 100644 --- a/plugins/links/hosted/video_src-hosted.js +++ b/plugins/links/hosted/video_src-hosted.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: 'video_src', diff --git a/plugins/links/iframely-link.js b/plugins/links/iframely-link.js index d826b773f..36cc1231d 100644 --- a/plugins/links/iframely-link.js +++ b/plugins/links/iframely-link.js @@ -1,7 +1,7 @@ -var utils = require('./utils'); -var _ = require('underscore'); +import utils from './utils.js'; +import * as _ from 'underscore'; -module.exports = { +export default { getLinks: function(meta, whitelistRecord) { return _.flatten(_.keys(meta).map(function(key) { diff --git a/plugins/links/image_src.js b/plugins/links/image_src.js index 971d0d72b..af29792bf 100644 --- a/plugins/links/image_src.js +++ b/plugins/links/image_src.js @@ -1,6 +1,6 @@ -var utils = require('./utils'); +import utils from './utils.js'; -module.exports = { +export default { getLink: function(meta) { return utils.getImageLink('image_src', meta); diff --git a/plugins/links/logo.js b/plugins/links/logo.js index b3d51a834..902cb7ca7 100644 --- a/plugins/links/logo.js +++ b/plugins/links/logo.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(meta) { diff --git a/plugins/links/ms-doc-viewer.js b/plugins/links/ms-doc-viewer.js index f664db77e..77cffa9af 100644 --- a/plugins/links/ms-doc-viewer.js +++ b/plugins/links/ms-doc-viewer.js @@ -1,6 +1,6 @@ // https://support.office.com/en-gb/article/View-Office-documents-online-1cc2ea26-0f7b-41f7-8e0e-6461a104544e?ui=en-US&rs=en-GB&ad=GB -module.exports = { +export default { getLink: function(url, __nonHtmlContentData, options) { diff --git a/plugins/links/oembed-icon.js b/plugins/links/oembed-icon.js index 5be21e978..34e6e7033 100644 --- a/plugins/links/oembed-icon.js +++ b/plugins/links/oembed-icon.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(oembed) { diff --git a/plugins/custom/oembed-iframe.js b/plugins/links/oembed-iframe.js similarity index 90% rename from plugins/custom/oembed-iframe.js rename to plugins/links/oembed-iframe.js index 464093447..2e997eec5 100644 --- a/plugins/custom/oembed-iframe.js +++ b/plugins/links/oembed-iframe.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: "iframe", diff --git a/plugins/links/oembed-photo-html.js b/plugins/links/oembed-photo-html.js index 54e227957..113099be3 100644 --- a/plugins/links/oembed-photo-html.js +++ b/plugins/links/oembed-photo-html.js @@ -1,6 +1,4 @@ -var cheerio = require('cheerio'); - -module.exports = { +export default { // this is the case of oembed photo or image, but with the html field // ex: @@ -19,17 +17,11 @@ module.exports = { }; - var $container = cheerio('<div>'); - try { - $container.html(oembed.html5 || oembed.html); - } catch (ex) {} - - var $iframe = $container.find('iframe'); - + var iframe = oembed.getIframe(); // if embed code contains <iframe>, return src - if ($iframe.length == 1) { - image.href = $iframe.attr('src'); + if (iframe.src) { + image.href = iframe.src; } else { image.html = oembed.html || oembed.html5; // will render in an iframe } diff --git a/plugins/links/oembed-photo.js b/plugins/links/oembed-photo.js index 53734bffb..d51af72e7 100644 --- a/plugins/links/oembed-photo.js +++ b/plugins/links/oembed-photo.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(oembed, whitelistRecord, url) { diff --git a/plugins/links/oembed-rich.js b/plugins/links/oembed-rich.js index 3a214f9bd..b6817f16c 100644 --- a/plugins/links/oembed-rich.js +++ b/plugins/links/oembed-rich.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(oembed, whitelistRecord, url) { diff --git a/plugins/links/oembed-thumbnail.js b/plugins/links/oembed-thumbnail.js index d9dee6097..5bb677fb3 100644 --- a/plugins/links/oembed-thumbnail.js +++ b/plugins/links/oembed-thumbnail.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(oembed) { if (oembed.thumbnail_url) { diff --git a/plugins/links/oembed-video.js b/plugins/links/oembed-video.js index ef5f180e8..aed329564 100644 --- a/plugins/links/oembed-video.js +++ b/plugins/links/oembed-video.js @@ -1,7 +1,8 @@ -var cheerio = require('cheerio'); -var entities = require('entities'); +import cheerio from 'cheerio'; -module.exports = { +import * as entities from 'entities'; + +export default { getLink: function(oembed, whitelistRecord, url) { @@ -13,25 +14,11 @@ module.exports = { rel:[CONFIG.R.oembed, CONFIG.R.player] }; - - // allow encoded entities if they start from $lt; - // ex.: http://www.nfb.ca/film/wild_life/ - var html = oembed.html5 || oembed.html; - if (/^</i.test(html)) { - html = entities.decodeHTML(html); - } - - var $container = cheerio('<div>'); - try { - $container.html(html); - } catch (ex) {} - - var $iframe = $container.find('iframe'); - + var iframe = oembed.getIframe(); // if embed code contains <iframe>, return src - if ($iframe.length == 1 && $iframe.attr('src')) { - player.href = $iframe.attr('src'); + if (iframe && iframe.src) { + player.href = iframe.src; if (whitelistRecord.isAllowed('oembed.video', 'ssl')) { player.href = player.href.replace(/^http:\/\//i, '//'); @@ -51,7 +38,7 @@ module.exports = { } } else { - player.html = html; // will render in an iframe + player.html = oembed.html; // will render in an iframe player.type = CONFIG.T.text_html; } @@ -75,8 +62,8 @@ module.exports = { player.rel.push(CONFIG.R.html5); } - if ($iframe.length == 1 && $iframe.attr('allow')) { - player.rel = player.rel.concat($iframe.attr('allow').replace(/autoplay;?\s?\*?/ig, '').split(/\s?\*?;\s?\*?/g)); + if (iframe && iframe.allow) { + player.rel = player.rel.concat(iframe.allow.replace(/autoplay;?\s?\*?/ig, '').split(/\s?\*?;\s?\*?/g)); } return player; @@ -97,7 +84,6 @@ module.exports = { /* tests: [ "http://sports.pixnet.net/album/video/183041064", - "http://video.yandex.ua/users/enema-bandit/view/11/?ncrnd=4917#hq" ] */ }; \ No newline at end of file diff --git a/plugins/links/og-image.js b/plugins/links/og-image.js index 599bd7276..f7c9ab9c5 100644 --- a/plugins/links/og-image.js +++ b/plugins/links/og-image.js @@ -1,4 +1,4 @@ -var _ = require("underscore"); +import * as _ from "underscore"; var rel = [CONFIG.R.thumbnail, CONFIG.R.og]; @@ -25,7 +25,7 @@ function getImageLinks(image) { return images; } -module.exports = { +export default { getLinks: function(og) { diff --git a/plugins/links/og-video.js b/plugins/links/og-video.js index a7ea76e8c..e690773f7 100644 --- a/plugins/links/og-video.js +++ b/plugins/links/og-video.js @@ -1,5 +1,5 @@ -var _ = require("underscore"); -var utils = require('./utils'); +import * as _ from "underscore"; +import utils from './utils.js'; function getVideoLinks(video, whitelistRecord) { @@ -48,7 +48,7 @@ function getVideoLinks(video, whitelistRecord) { return players; } -module.exports = { +export default { getLinks: function(og, whitelistRecord) { diff --git a/plugins/links/parsely-image.js b/plugins/links/parsely-image.js index 52ef44453..67f6976bc 100644 --- a/plugins/links/parsely-image.js +++ b/plugins/links/parsely-image.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(parsely) { return { diff --git a/plugins/links/prerender/checkAppFlag.js b/plugins/links/prerender/checkAppFlag.js index 10377b008..2748ac785 100644 --- a/plugins/links/prerender/checkAppFlag.js +++ b/plugins/links/prerender/checkAppFlag.js @@ -1,6 +1,6 @@ -const utils = require('./utils'); +import utils from './utils.js'; -module.exports = { +export default { provides: '__appFlag', diff --git a/plugins/links/prerender/prerender.js b/plugins/links/prerender/prerender.js index a4a432661..7bd0f9102 100644 --- a/plugins/links/prerender/prerender.js +++ b/plugins/links/prerender/prerender.js @@ -1,13 +1,12 @@ -const core = require('../../../lib/core'); -const utils = require('./utils'); +import utils from './utils.js'; -module.exports = { +export default { highestPriority: true, provides: ['appUriData', 'whenPrerender'], - getData: function(url, __appFlag, options, meta, cb) { + getData: function(url, __appFlag, iframelyRun, options, meta, cb) { if (CONFIG.PRERENDER && CONFIG.PRERENDER_URL && options.user_agent === CONFIG.FB_USER_AGENT) { @@ -17,7 +16,7 @@ module.exports = { refresh: true }}; - core.run(prerenderUrl, options2, function(error, data) { + iframelyRun(prerenderUrl, options2, function(error, data) { var title = data && data.meta && ((data.meta.og && data.meta.og.title) || (data.meta.twitter && data.meta.twitter.title) || data.meta.title || data.meta['html-title']); diff --git a/plugins/links/prerender/react-app-fb-fallback.js b/plugins/links/prerender/react-app-fb-fallback.js index af3b039e3..03b3ed171 100644 --- a/plugins/links/prerender/react-app-fb-fallback.js +++ b/plugins/links/prerender/react-app-fb-fallback.js @@ -1,13 +1,12 @@ -const core = require('../../../lib/core'); -const utils = require('./utils'); +import utils from './utils.js'; -module.exports = { +export default { highestPriority: true, provides: ['appUriData', 'whenReact'], - getData: function(url, __appFlag, options, cb) { + getData: function(url, __appFlag, iframelyRun, options, cb) { if (options.user_agent === CONFIG.FB_USER_AGENT) { return cb(); @@ -19,7 +18,7 @@ module.exports = { user_agent: CONFIG.FB_USER_AGENT }}; - core.run(url, options2, function(error, data) { + iframelyRun(url, options2, function(error, data) { if (data && data.meta && utils.maybeApp(data.meta)) { return cb({ diff --git a/plugins/links/prerender/utils.js b/plugins/links/prerender/utils.js index 9f1717238..b53e0a4da 100644 --- a/plugins/links/prerender/utils.js +++ b/plugins/links/prerender/utils.js @@ -1,4 +1,4 @@ -module.exports = { +export default { notPlugin: true, diff --git a/plugins/links/sailthru.js b/plugins/links/sailthru.js index e5f026502..8c72e8696 100644 --- a/plugins/links/sailthru.js +++ b/plugins/links/sailthru.js @@ -1,6 +1,6 @@ -var utils = require('./utils'); +import utils from './utils.js'; -module.exports = { +export default { provides: 'sailthru', diff --git a/plugins/links/thumbnail.js b/plugins/links/thumbnail.js index 086087f64..604f517d5 100644 --- a/plugins/links/thumbnail.js +++ b/plugins/links/thumbnail.js @@ -1,6 +1,6 @@ -var utils = require('./utils'); +import utils from './utils.js'; -module.exports = { +export default { getLink: function(meta) { if (!meta.og || !meta.og.image) { diff --git a/plugins/links/twitter-image.js b/plugins/links/twitter-image.js index 023a3c665..01dc7fd2b 100644 --- a/plugins/links/twitter-image.js +++ b/plugins/links/twitter-image.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLinks: function(twitter, meta, whitelistRecord) { diff --git a/plugins/links/twitter-player.js b/plugins/links/twitter-player.js index 6300ee611..e4376936e 100644 --- a/plugins/links/twitter-player.js +++ b/plugins/links/twitter-player.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function(twitter, whitelistRecord) { diff --git a/plugins/links/twitter-stream.js b/plugins/links/twitter-stream.js index 82c4a1b2b..bd9922f66 100644 --- a/plugins/links/twitter-stream.js +++ b/plugins/links/twitter-stream.js @@ -1,4 +1,4 @@ -var _ = require("underscore"); +import * as _ from "underscore"; function getStreamLinks(twitter, stream, whitelistRecord) { @@ -31,7 +31,7 @@ function getStreamLinks(twitter, stream, whitelistRecord) { return player; } -module.exports = { +export default { getLink: function(twitter, whitelistRecord) { diff --git a/plugins/links/utils.js b/plugins/links/utils.js index 21be78d8f..b4ce60a4e 100644 --- a/plugins/links/utils.js +++ b/plugins/links/utils.js @@ -1,4 +1,4 @@ -var _ = require('underscore'); +import * as _ from 'underscore'; var ALLOWED_TYPES = {}; @@ -6,7 +6,7 @@ _.values(CONFIG.T).forEach(function(v) { ALLOWED_TYPES[v] = true; }); -module.exports = { +export default { notPlugin: true, diff --git a/plugins/links/video_src.js b/plugins/links/video_src.js index 2c9fb4e63..9a135f55c 100644 --- a/plugins/links/video_src.js +++ b/plugins/links/video_src.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getLink: function (meta, whitelistRecord) { diff --git a/plugins/meta/amphtml.js b/plugins/meta/amphtml.js index 33aa89365..c70244f69 100644 --- a/plugins/meta/amphtml.js +++ b/plugins/meta/amphtml.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { return { diff --git a/plugins/meta/author.js b/plugins/meta/author.js index e903f266e..3b80995ce 100644 --- a/plugins/meta/author.js +++ b/plugins/meta/author.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/canonical.js b/plugins/meta/canonical.js index 619e55616..0699849f1 100644 --- a/plugins/meta/canonical.js +++ b/plugins/meta/canonical.js @@ -1,4 +1,4 @@ -module.exports = { +export default { highestPriority: true, diff --git a/plugins/meta/category.js b/plugins/meta/category.js index 688f98988..dfc825a62 100644 --- a/plugins/meta/category.js +++ b/plugins/meta/category.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { return { diff --git a/plugins/meta/copyright.js b/plugins/meta/copyright.js index 6113772d5..b82a95318 100644 --- a/plugins/meta/copyright.js +++ b/plugins/meta/copyright.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { return { diff --git a/plugins/meta/date.js b/plugins/meta/date.js index 21a15e7d6..15cbaf061 100644 --- a/plugins/meta/date.js +++ b/plugins/meta/date.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta, url) { diff --git a/plugins/meta/dc-title.js b/plugins/meta/dc-title.js index 954c8ec4a..3727357e8 100644 --- a/plugins/meta/dc-title.js +++ b/plugins/meta/dc-title.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/dc.js b/plugins/meta/dc.js index efeb73c5e..2ea88bc8a 100644 --- a/plugins/meta/dc.js +++ b/plugins/meta/dc.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/description-from-p-tag.js b/plugins/meta/description-from-p-tag.js index ace4bbf34..9bd81dfba 100644 --- a/plugins/meta/description-from-p-tag.js +++ b/plugins/meta/description-from-p-tag.js @@ -1,6 +1,6 @@ -var decodeHTML5 = require('entities').decodeHTML5; +import { decodeHTML5 } from 'entities'; -module.exports = { +export default { getMeta: function(cheerio, decode, __allowPTagDescription) { // Get the text from the first <p> tag that's not in a header diff --git a/plugins/meta/description.js b/plugins/meta/description.js index 8893a4f36..71c01d0b9 100644 --- a/plugins/meta/description.js +++ b/plugins/meta/description.js @@ -1,4 +1,4 @@ -module.exports = { +export default { lowestPriority: true, diff --git a/plugins/meta/embedurl-meta.js b/plugins/meta/embedurl-meta.js index c730c88c6..40a6d78b3 100644 --- a/plugins/meta/embedurl-meta.js +++ b/plugins/meta/embedurl-meta.js @@ -1,4 +1,4 @@ -module.exports = { +export default { lowestPriority: true, diff --git a/plugins/meta/geo-url.js b/plugins/meta/geo-url.js index a222927e4..3c5080a81 100644 --- a/plugins/meta/geo-url.js +++ b/plugins/meta/geo-url.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/html-title.js b/plugins/meta/html-title.js index f56dd7085..8f6ff3b13 100644 --- a/plugins/meta/html-title.js +++ b/plugins/meta/html-title.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { return { diff --git a/plugins/meta/keywords.js b/plugins/meta/keywords.js index eea548b85..85bed842f 100644 --- a/plugins/meta/keywords.js +++ b/plugins/meta/keywords.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/ld-article-keywords.js b/plugins/meta/ld-article-keywords.js index be11d70d5..d77b6c4a0 100644 --- a/plugins/meta/ld-article-keywords.js +++ b/plugins/meta/ld-article-keywords.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(ld) { diff --git a/plugins/meta/ld-article.js b/plugins/meta/ld-article.js index 07e17902c..068e273b5 100644 --- a/plugins/meta/ld-article.js +++ b/plugins/meta/ld-article.js @@ -1,6 +1,7 @@ -const cheerio = require('cheerio'); +import cheerio from 'cheerio'; -module.exports = { + +export default { getMeta: function(ld) { function clean(field) { @@ -14,8 +15,8 @@ module.exports = { return $container.text(); } } - if (ld.newsarticle) { + if (ld.newsarticle) { return { title: clean(ld.newsarticle.headline), category: clean(ld.newsarticle.articlesection), diff --git a/plugins/meta/ld-product.js b/plugins/meta/ld-product.js index 8928aaedd..c2e7df3b9 100644 --- a/plugins/meta/ld-product.js +++ b/plugins/meta/ld-product.js @@ -1,6 +1,7 @@ -const cheerio = require('cheerio'); +import cheerio from 'cheerio'; -module.exports = { + +export default { highestPriority: true, diff --git a/plugins/meta/media-detector.js b/plugins/meta/media-detector.js index f465005bd..0f8e5ffe3 100644 --- a/plugins/meta/media-detector.js +++ b/plugins/meta/media-detector.js @@ -1,6 +1,6 @@ -const utils = require('../links/utils'); +import utils from '../links/utils.js'; -module.exports = { +export default { lowestPriority: true, diff --git a/plugins/meta/meta-music.js b/plugins/meta/meta-music.js index 464e1114c..31efeba6e 100644 --- a/plugins/meta/meta-music.js +++ b/plugins/meta/meta-music.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { if (meta.music) { diff --git a/plugins/meta/meta-title.js b/plugins/meta/meta-title.js index 03b214430..f99d26ce3 100644 --- a/plugins/meta/meta-title.js +++ b/plugins/meta/meta-title.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/noindex-robots.js b/plugins/meta/noindex-robots.js index 8f40e86d7..a96a997ae 100644 --- a/plugins/meta/noindex-robots.js +++ b/plugins/meta/noindex-robots.js @@ -1,6 +1,6 @@ -var oembedUtils = require('../../lib/plugins/system/oembed/oembedUtils'); +import * as oembedUtils from '../../lib/plugins/system/oembed/oembedUtils.js'; -module.exports = { +export default { getData: function(url, meta, __noOembedLinks, cb) { diff --git a/plugins/meta/noscript-redirect.js b/plugins/meta/noscript-redirect.js index 4f2857271..b5093565b 100644 --- a/plugins/meta/noscript-redirect.js +++ b/plugins/meta/noscript-redirect.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getData: function(url, meta, cb) { return cb( diff --git a/plugins/meta/oembed-author.js b/plugins/meta/oembed-author.js index 20be0f028..2acb39c92 100644 --- a/plugins/meta/oembed-author.js +++ b/plugins/meta/oembed-author.js @@ -1,4 +1,4 @@ -module.exports = { +export default { highestPriority: true, diff --git a/plugins/meta/oembed-canonical.js b/plugins/meta/oembed-canonical.js index 6c6aacb85..f4bb390cc 100644 --- a/plugins/meta/oembed-canonical.js +++ b/plugins/meta/oembed-canonical.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(oembed) { if (oembed.type != "photo") { diff --git a/plugins/meta/oembed-description.js b/plugins/meta/oembed-description.js index 39cfd66d8..d3543b88b 100644 --- a/plugins/meta/oembed-description.js +++ b/plugins/meta/oembed-description.js @@ -1,6 +1,7 @@ -var cheerio = require('cheerio'); +import cheerio from 'cheerio'; -module.exports = { + +export default { getMeta: function(oembed) { if (oembed.description) { diff --git a/plugins/meta/oembed-duration.js b/plugins/meta/oembed-duration.js index 7d6515f81..db94d3dd9 100644 --- a/plugins/meta/oembed-duration.js +++ b/plugins/meta/oembed-duration.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(oembed) { return { diff --git a/plugins/meta/oembed-license.js b/plugins/meta/oembed-license.js index 70316e2d3..10cd9a327 100644 --- a/plugins/meta/oembed-license.js +++ b/plugins/meta/oembed-license.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(oembed) { return { diff --git a/plugins/meta/oembed-product.js b/plugins/meta/oembed-product.js index bee73397d..29db456e3 100644 --- a/plugins/meta/oembed-product.js +++ b/plugins/meta/oembed-product.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(oembed) { diff --git a/plugins/meta/oembed-site.js b/plugins/meta/oembed-site.js index 4d52bd80c..f4f9dcc29 100644 --- a/plugins/meta/oembed-site.js +++ b/plugins/meta/oembed-site.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(oembed) { return { diff --git a/plugins/meta/oembed-title.js b/plugins/meta/oembed-title.js index b6edfc280..012a16d22 100644 --- a/plugins/meta/oembed-title.js +++ b/plugins/meta/oembed-title.js @@ -1,4 +1,4 @@ -module.exports = { +export default { // avoid wordpress pages lowestPriority: true, diff --git a/plugins/meta/og-article.js b/plugins/meta/og-article.js index 2a8abbccc..b563e4f61 100644 --- a/plugins/meta/og-article.js +++ b/plugins/meta/og-article.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/og-date.js b/plugins/meta/og-date.js index 1c99ab726..6a6b46f69 100644 --- a/plugins/meta/og-date.js +++ b/plugins/meta/og-date.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(og) { diff --git a/plugins/meta/og-description.js b/plugins/meta/og-description.js index 8012f415a..5654db4ce 100644 --- a/plugins/meta/og-description.js +++ b/plugins/meta/og-description.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(og) { diff --git a/plugins/meta/og-location.js b/plugins/meta/og-location.js index b66b99e51..b832ee4ca 100644 --- a/plugins/meta/og-location.js +++ b/plugins/meta/og-location.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(og) { diff --git a/plugins/meta/og-product.js b/plugins/meta/og-product.js index 8b84fd0e1..e0514af6a 100644 --- a/plugins/meta/og-product.js +++ b/plugins/meta/og-product.js @@ -1,4 +1,4 @@ -module.exports = { +export default { // https://developers.pinterest.com/docs/rich-pins/products/? getMeta: function(og, meta) { diff --git a/plugins/meta/og-site.js b/plugins/meta/og-site.js index c2c7a34a7..8d648edce 100644 --- a/plugins/meta/og-site.js +++ b/plugins/meta/og-site.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(og) { diff --git a/plugins/meta/og-title.js b/plugins/meta/og-title.js index e0e32ae56..fa5a67fc3 100644 --- a/plugins/meta/og-title.js +++ b/plugins/meta/og-title.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(og) { diff --git a/plugins/meta/og-video-duration.js b/plugins/meta/og-video-duration.js index c67b40930..61eaf8a1d 100644 --- a/plugins/meta/og-video-duration.js +++ b/plugins/meta/og-video-duration.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(og) { diff --git a/plugins/meta/parsely.js b/plugins/meta/parsely.js index e32682743..f9f82c8bc 100644 --- a/plugins/meta/parsely.js +++ b/plugins/meta/parsely.js @@ -1,4 +1,4 @@ -module.exports = { +export default { highestPriority: true, diff --git a/plugins/meta/shortlink.js b/plugins/meta/shortlink.js index 3e041f7a8..269304106 100644 --- a/plugins/meta/shortlink.js +++ b/plugins/meta/shortlink.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { var s = meta.shorturl || meta.short_url || meta.shortlink; diff --git a/plugins/meta/site.js b/plugins/meta/site.js index bd1003934..5a7ed63d9 100644 --- a/plugins/meta/site.js +++ b/plugins/meta/site.js @@ -1,4 +1,4 @@ -module.exports = { +export default { lowestPriority: true, diff --git a/plugins/meta/theme-color.js b/plugins/meta/theme-color.js index f92f68973..fb23c8634 100644 --- a/plugins/meta/theme-color.js +++ b/plugins/meta/theme-color.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/twitter-description.js b/plugins/meta/twitter-description.js index ecb434754..aec8d7689 100644 --- a/plugins/meta/twitter-description.js +++ b/plugins/meta/twitter-description.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(twitter) { diff --git a/plugins/meta/twitter-labels.js b/plugins/meta/twitter-labels.js index 1ba25f6f1..b38146fc3 100644 --- a/plugins/meta/twitter-labels.js +++ b/plugins/meta/twitter-labels.js @@ -1,4 +1,4 @@ -module.exports = { +export default { lowestPriority: true, diff --git a/plugins/meta/twitter-site.js b/plugins/meta/twitter-site.js index 57deb18ba..60c331145 100644 --- a/plugins/meta/twitter-site.js +++ b/plugins/meta/twitter-site.js @@ -1,4 +1,4 @@ -module.exports = { +export default { lowestPriority: true, diff --git a/plugins/meta/twitter-title.js b/plugins/meta/twitter-title.js index 44f3ecd53..1db8a7262 100644 --- a/plugins/meta/twitter-title.js +++ b/plugins/meta/twitter-title.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(twitter) { diff --git a/plugins/meta/video-duration.js b/plugins/meta/video-duration.js index 6974a47a0..0757023ce 100644 --- a/plugins/meta/video-duration.js +++ b/plugins/meta/video-duration.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/video.js b/plugins/meta/video.js index 5267040dd..dc83691d9 100644 --- a/plugins/meta/video.js +++ b/plugins/meta/video.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { diff --git a/plugins/meta/yandex-ovs.js b/plugins/meta/yandex-ovs.js index 5553bfc9a..034c32f92 100644 --- a/plugins/meta/yandex-ovs.js +++ b/plugins/meta/yandex-ovs.js @@ -1,4 +1,4 @@ -module.exports = { +export default { provides: 'ovs', diff --git a/server.js b/server.js index 1ba753503..6e969f166 100644 --- a/server.js +++ b/server.js @@ -1,5 +1,6 @@ -var sysUtils = require('./utils'); -var app = require('./app'); +import * as sysUtils from './utils.js'; +import app from './app.js'; +import * as https from 'https'; var server = app.listen(process.env.PORT || CONFIG.port, process.env.HOST || CONFIG.host, function(){ console.log('\niframely is running on ' + server.address().address + ':' + server.address().port); @@ -7,7 +8,7 @@ var server = app.listen(process.env.PORT || CONFIG.port, process.env.HOST || CON }); if (CONFIG.ssl) { - require('https').createServer(CONFIG.ssl, app).listen(CONFIG.ssl.port); + https.createServer(CONFIG.ssl, app).listen(CONFIG.ssl.port); } console.log(''); @@ -15,13 +16,12 @@ console.log(' - support@iframely.com - if you need help'); console.log(' - twitter.com/iframely - news & updates'); console.log(' - github.com/itteco/iframely - star & contribute'); +import { GracefulServer } from 'graceful-cluster'; + if (!CONFIG.DEBUG) { - var GracefulServer = require('graceful-cluster').GracefulServer; new GracefulServer({ server: server, log: sysUtils.log, shutdownTimeout: CONFIG.SHUTDOWN_TIMEOUT }); -} - -module.exports = server; +} \ No newline at end of file diff --git a/test/core-plugins.js b/test/core-plugins.js index 4191ad864..86195b445 100644 --- a/test/core-plugins.js +++ b/test/core-plugins.js @@ -1,10 +1,11 @@ -var assert = require('chai').assert +import { assert as assert } from 'chai' -global.CONFIG = require('../config'); +import config from '../config'; +global.CONFIG = config; -var iframely = require('../lib/core').getPluginData; -var findWhitelistRecordFor = require('../lib/whitelist').findWhitelistRecordFor; -var utils = require('../lib/utils'); +import { getPluginData as iframely } from '../lib/core'; +import { findWhitelistRecordFor as findWhitelistRecordFor } from '../lib/whitelist'; +import * as utils from '../lib/utils.js'; function assertOembed(oembed) { assert.isObject(oembed); diff --git a/test/custom_plugins.js b/test/custom_plugins.js index 72aef2a18..b8b280f9f 100644 --- a/test/custom_plugins.js +++ b/test/custom_plugins.js @@ -1,9 +1,8 @@ 'use strict'; - -var request = require('supertest'); -var ServerMock = require('mock-http-server'); -var chai = require('chai'); -var async = require('async'); +import * as request from 'supertest'; +import * as ServerMock from 'mock-http-server'; +import * as chai from 'chai'; +import * as async from 'async'; var server; @@ -15,7 +14,7 @@ function invalidateRequireCache () { var startWithENV = function (env, cb) { process.env.NODE_ENV = env; - var app = require('../app'); +import * as app from '../app.js'; server = app.listen(process.env.PORT, cb); }; diff --git a/test/e2e.js b/test/e2e.js index 63d6e3a7d..8173f50d1 100644 --- a/test/e2e.js +++ b/test/e2e.js @@ -1,9 +1,8 @@ 'use strict'; - -var request = require('supertest'); -var ServerMock = require('mock-http-server'); -var chai = require('chai'); -var async = require('async'); +import * as request from 'supertest'; +import * as ServerMock from 'mock-http-server'; +import * as chai from 'chai'; +import * as async from 'async'; describe('meta endpoint', function() { @@ -16,7 +15,7 @@ describe('meta endpoint', function() { var server; beforeEach(function(done) { - var app = require('../app'); +import * as app from '../app.js'; server = app.listen(process.env.PORT, function() { targetMockedServer.start(done); }); diff --git a/test/fixtures/custom-plugins/domains/test.com.js b/test/fixtures/custom-plugins/domains/test.com.js index 5772a1adf..0bdebfebc 100644 --- a/test/fixtures/custom-plugins/domains/test.com.js +++ b/test/fixtures/custom-plugins/domains/test.com.js @@ -1,6 +1,4 @@ -var utils = require('../../../../utils'); - -module.exports = { +export default { re: /testok/i, diff --git a/test/fixtures/custom-plugins/meta/html-title.js b/test/fixtures/custom-plugins/meta/html-title.js index 11379bedc..d0b682d25 100644 --- a/test/fixtures/custom-plugins/meta/html-title.js +++ b/test/fixtures/custom-plugins/meta/html-title.js @@ -1,4 +1,4 @@ -module.exports = { +export default { getMeta: function(meta) { return { diff --git a/tester.js b/tester.js new file mode 100644 index 000000000..8d6bfef0e --- /dev/null +++ b/tester.js @@ -0,0 +1 @@ +import './modules/tests-ui/tester.js'; \ No newline at end of file diff --git a/utils.js b/utils.js index 18d8e9f4d..0a31c315b 100644 --- a/utils.js +++ b/utils.js @@ -1,21 +1,23 @@ -(function() { + import * as async from 'async'; + import { cache } from './lib/cache.js'; + import * as ejs from 'ejs'; + import * as fs from 'fs'; + import * as crypto from 'crypto'; + import * as _ from 'underscore'; + import * as urlLib from 'url'; - global.CONFIG = require('./config'); + import log from './logging.js'; + export { log }; + import * as whitelist from './lib/whitelist.js'; + import * as pluginLoader from './lib/loader/pluginLoader.js'; - var async = require('async'); - var cache = require('./lib/cache'); - var ejs = require('ejs'); - var fs = require('fs'); - var crypto = require('crypto'); - var _ = require('underscore'); - var urlLib = require('url'); + import { fileURLToPath } from 'url'; + import { dirname } from 'path'; - var log = exports.log = require('./logging').log; + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); - var whitelist = require('./lib/whitelist'); - var pluginLoader = require('./lib/loader/pluginLoader'); - - function NotFound(message, messages) { + export function NotFound(message, messages) { if (typeof message === 'object') { this.meta = message; @@ -33,9 +35,7 @@ NotFound.prototype.__proto__ = Error.prototype; - exports.NotFound = NotFound; - - function HttpError(code, message, messages) { + export function HttpError(code, message, messages) { Error.call(this); //super constructor Error.captureStackTrace(this, this.constructor); //super helper method to include stack trace in error object @@ -49,9 +49,9 @@ HttpError.prototype.__proto__ = Error.prototype; - exports.HttpError = HttpError; - - var version = require('./package.json').version; + import { readFile } from 'fs/promises'; + const json = JSON.parse(await readFile(new URL('./package.json', import.meta.url))); + var version = json.version; var etag = function(value) { return '"' + crypto.createHash('md5').update(value).digest("hex") + '"'; @@ -158,7 +158,7 @@ cache.set('urlcache:' + version + (linkValidationKey || '') + ':' + url, data, {ttl: ttl}); } - exports.cacheMiddleware = function(req, res, next) { + export function cacheMiddleware(req, res, next) { async.waterfall([ @@ -334,5 +334,3 @@ next(); }); }; - -})(); diff --git a/yarn.lock b/yarn.lock index d169cc85f..28a465e20 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,25 +2,62 @@ # yarn lockfile v1 +"@adobe/helix-fetch@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@adobe/helix-fetch/-/helix-fetch-3.0.0.tgz#89c9c01ca3894375d6ca6cefa2ce974bcbc37f46" + integrity sha512-yvkOBwXLYchH/ql/KZ0Rp7QLuw0J38q0ZgOiPeF4ZPrBFfyJD6HmAX5AjvHEaofOyPKIJmlZb5KD+6kjE+01kg== + dependencies: + debug "4.3.2" + http-cache-semantics "4.1.0" + lru-cache "6.0.0" + +"@nodelib/fs.scandir@2.1.5": + version "2.1.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" + integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== + dependencies: + "@nodelib/fs.stat" "2.0.5" + run-parallel "^1.1.9" + +"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" + integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== + +"@nodelib/fs.walk@^1.2.3": + version "1.2.8" + resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" + integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== + dependencies: + "@nodelib/fs.scandir" "2.1.5" + fastq "^1.6.0" + "@types/bson@*": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.0.3.tgz#30889d2ffde6262abbe38659364c631454999fbf" - integrity sha512-mVRvYnTOZJz3ccpxhr3wgxVmSeiYinW+zlzQz3SXWaJmD1DuL05Jeq7nKw3SnbKmbleW5qrLG5vdyWe/A9sXhw== + version "4.2.0" + resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337" + integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg== + dependencies: + bson "*" + +"@types/bson@1.x || 4.0.x": + version "4.0.5" + resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.0.5.tgz#9e0e1d1a6f8866483f96868a9b33bc804926b1fc" + integrity sha512-vVLwMUqhYJSQ/WKcE60eFqcyuWse5fGH+NMAXHuKrUAPoryq3ATxk5o4bgYNtg5aOM4APVg7Hnb3ASqUYG0PKg== dependencies: "@types/node" "*" "@types/mongodb@^3.5.27": - version "3.6.12" - resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.12.tgz#727960d34f35054d2f2ce68909e16094f742d935" - integrity sha512-49aEzQD5VdHPxyd5dRyQdqEveAg9LanwrH8RQipnMuulwzKmODXIZRp0umtxi1eBUfEusRkoy8AVOMr+kVuFog== + version "3.6.20" + resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.20.tgz#b7c5c580644f6364002b649af1c06c3c0454e1d2" + integrity sha512-WcdpPJCakFzcWWD9juKoZbRtQxKIMYF/JIAM4JrNHrMcnJL6/a2NWjXxW7fo9hxboxxkg+icff8d7+WIEvKgYQ== dependencies: "@types/bson" "*" "@types/node" "*" "@types/node@*": - version "15.0.2" - resolved "https://registry.yarnpkg.com/@types/node/-/node-15.0.2.tgz#51e9c0920d1b45936ea04341aa3e2e58d339fb67" - integrity sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA== + version "16.11.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.1.tgz#2e50a649a50fc403433a14f829eface1a3443e97" + integrity sha512-PYGcJHL9mwl1Ek3PLiYgyEKtwTMmkMw4vbiyz/ps3pfdRYLVv+SN7qHVAImrjdAXxgluDEw6Ph4lyv+m9UpRmA== abbrev@1: version "1.1.1" @@ -50,11 +87,6 @@ ajv@^6.12.3: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= - ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -62,7 +94,7 @@ ansi-styles@^3.2.1: dependencies: color-convert "^1.9.0" -anymatch@~3.1.1: +anymatch@~3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== @@ -70,19 +102,6 @@ anymatch@~3.1.1: normalize-path "^3.0.0" picomatch "^2.0.4" -aproba@^1.0.3: - version "1.2.0" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" - integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== - -are-we-there-yet@~1.1.2: - version "1.1.5" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" - integrity sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w== - dependencies: - delegates "^1.0.0" - readable-stream "^2.0.6" - array-flatten@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" @@ -93,6 +112,11 @@ array-indexofobject@~0.0.1: resolved "https://registry.yarnpkg.com/array-indexofobject/-/array-indexofobject-0.0.1.tgz#aaa128e62c9b3c358094568c219ff64fe489d42a" integrity sha1-qqEo5iybPDWAlFaMIZ/2T+SJ1Co= +array-union@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-3.0.1.tgz#da52630d327f8b88cfbfb57728e2af5cd9b6b975" + integrity sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw== + asn1@~0.2.3: version "0.2.4" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" @@ -167,15 +191,6 @@ bl@^2.2.1: readable-stream "^2.3.5" safe-buffer "^5.1.1" -bl@^4.0.3: - version "4.1.0" - resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" - integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== - dependencies: - buffer "^5.5.0" - inherits "^2.0.4" - readable-stream "^3.4.0" - bluebird@3.5.1: version "3.5.1" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" @@ -210,7 +225,7 @@ brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -braces@~3.0.2: +braces@^3.0.1, braces@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== @@ -222,6 +237,13 @@ browser-stdout@1.3.1: resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== +bson@*: + version "4.5.3" + resolved "https://registry.yarnpkg.com/bson/-/bson-4.5.3.tgz#de3783b357a407d935510beb1fbb285fef43bb06" + integrity sha512-qVX7LX79Mtj7B3NPLzCfBiCP6RAsjiV8N63DjlaVVpZW+PFoDTxQ4SeDbSpcqgE6mXksM5CAwZnXxxxn/XwC0g== + dependencies: + buffer "^5.6.0" + bson@^1.1.4: version "1.1.6" resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.6.tgz#fb819be9a60cd677e0853aee4ca712a785d6618a" @@ -232,7 +254,7 @@ buffer-shims@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" integrity sha1-mXjOMXOIxkmth5MCjDR37wRKi1E= -buffer@^5.5.0: +buffer@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -268,7 +290,7 @@ chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -cheerio@0.22.0: +cheerio@^0.22.0: version "0.22.0" resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-0.22.0.tgz#a9baa860a3f9b595a6b81b1a86873121ed3a269e" integrity sha1-qbqoYKP5tZWmuBsahocxIe06Jp4= @@ -291,35 +313,25 @@ cheerio@0.22.0: lodash.some "^4.4.0" chokidar@^3.3.1: - version "3.4.3" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.4.3.tgz#c1df38231448e45ca4ac588e6c79573ba6a57d5b" - integrity sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ== + version "3.5.2" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.2.tgz#dba3976fcadb016f66fd365021d91600d01c1e75" + integrity sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ== dependencies: - anymatch "~3.1.1" + anymatch "~3.1.2" braces "~3.0.2" - glob-parent "~5.1.0" + glob-parent "~5.1.2" is-binary-path "~2.1.0" is-glob "~4.0.1" normalize-path "~3.0.0" - readdirp "~3.5.0" + readdirp "~3.6.0" optionalDependencies: - fsevents "~2.1.2" - -chownr@^1.1.1: - version "1.1.4" - resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" - integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== + fsevents "~2.3.2" cluster-key-slot@^1.0.5: version "1.1.0" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== -code-point-at@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" - integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= - color-convert@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -369,11 +381,6 @@ connection-parse@0.0.x: resolved "https://registry.yarnpkg.com/connection-parse/-/connection-parse-0.0.7.tgz#18e7318aab06a699267372b10c5226d25a1c9a69" integrity sha1-GOcxiqsGppkmc3KxDFIm0locmmk= -console-control-strings@^1.0.0, console-control-strings@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" - integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= - content-disposition@0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd" @@ -401,11 +408,16 @@ cookiejar@^2.1.0: resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.2.tgz#dd8a235530752f988f9a0844f3fc589e3111125c" integrity sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA== -core-util-is@1.0.2, core-util-is@~1.0.0: +core-util-is@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +core-util-is@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" + integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== + css-select@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858" @@ -442,6 +454,13 @@ debug@3.1.0: dependencies: ms "2.0.0" +debug@4.3.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" + integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== + dependencies: + ms "2.1.2" + debug@^3.1.0: version "3.2.7" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" @@ -449,13 +468,6 @@ debug@^3.1.0: dependencies: ms "^2.1.1" -decompress-response@^4.2.0: - version "4.2.1" - resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986" - integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw== - dependencies: - mimic-response "^2.0.0" - deep-eql@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2" @@ -463,11 +475,6 @@ deep-eql@^0.1.3: dependencies: type-detect "0.1.1" -deep-extend@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" - integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== - deepmerge@^4.0.0: version "4.2.2" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" @@ -478,16 +485,16 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= -delegates@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" - integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= - -denque@^1.1.0, denque@^1.4.1, denque@^1.5.0: +denque@^1.1.0, denque@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de" integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ== +denque@^1.4.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" + integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== + depd@~1.1.1, depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" @@ -498,17 +505,36 @@ destroy@~1.0.4: resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= -detect-libc@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" - integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= - diff@3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== -dom-serializer@0, dom-serializer@~0.1.0: +dir-glob@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" + integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== + dependencies: + path-type "^4.0.0" + +dom-serializer@0: + version "0.2.2" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51" + integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g== + dependencies: + domelementtype "^2.0.1" + entities "^2.0.0" + +dom-serializer@^1.0.1: + version "1.3.2" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.3.2.tgz#6206437d32ceefaec7161803230c7a20bc1b4d91" + integrity sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig== + dependencies: + domelementtype "^2.0.1" + domhandler "^4.2.0" + entities "^2.0.0" + +dom-serializer@~0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0" integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA== @@ -516,11 +542,16 @@ dom-serializer@0, dom-serializer@~0.1.0: domelementtype "^1.3.0" entities "^1.1.1" -domelementtype@1, domelementtype@^1.3.0: +domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f" integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== +domelementtype@^2.0.1, domelementtype@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57" + integrity sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A== + domhandler@2.0: version "2.0.3" resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.0.3.tgz#889f8df626403af0788e29d66d5d5c6f7ebf0fd6" @@ -535,6 +566,13 @@ domhandler@^2.3.0: dependencies: domelementtype "1" +domhandler@^4.2.0, domhandler@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.2.tgz#e825d721d19a86b8c201a35264e226c678ee755f" + integrity sha512-PzE9aBMsdZO8TK4BnuJwH0QT41wgMbRzuZrHUcpYncEjmQazq8QEaBWgLG7ZyC/DAZKEgglpIA6j4Qn/HmxS3w== + dependencies: + domelementtype "^2.2.0" + domutils@1.1: version "1.1.6" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.1.6.tgz#bddc3de099b9a2efacc51c623f28f416ecc57485" @@ -542,7 +580,7 @@ domutils@1.1: dependencies: domelementtype "1" -domutils@1.5.1, domutils@^1.5.1: +domutils@1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" integrity sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8= @@ -550,6 +588,23 @@ domutils@1.5.1, domutils@^1.5.1: dom-serializer "0" domelementtype "1" +domutils@^1.5.1: + version "1.7.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" + integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg== + dependencies: + dom-serializer "0" + domelementtype "1" + +domutils@^2.8.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135" + integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A== + dependencies: + dom-serializer "^1.0.1" + domelementtype "^2.2.0" + domhandler "^4.2.0" + double-ended-queue@^2.1.0-0: version "2.1.0-0" resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" @@ -580,23 +635,31 @@ encodeurl@~1.0.1, encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= -end-of-stream@^1.1.0, end-of-stream@^1.4.1: - version "1.4.4" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== - dependencies: - once "^1.4.0" - entities@0: version "0.5.0" resolved "https://registry.yarnpkg.com/entities/-/entities-0.5.0.tgz#f611cb5ae221050e0012c66979503fd7ae19cc49" integrity sha1-9hHLWuIhBQ4AEsZpeVA/164ZzEk= -entities@1.1.1, entities@^1.1.1, entities@~1.1.1: +entities@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0" integrity sha1-blwtClYhtdra7O+AuQ7ftc13cvA= +entities@^1.1.1, entities@~1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56" + integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w== + +entities@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" + integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== + +entities@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/entities/-/entities-3.0.1.tgz#2b887ca62585e96db3903482d336c1006c3001d4" + integrity sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q== + escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" @@ -617,11 +680,6 @@ exit@~0.1.2: resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" integrity sha1-BjJjj42HfMghB9MKD/8aF8uhzQw= -expand-template@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" - integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== - express@^4.16.3: version "4.17.1" resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134" @@ -673,11 +731,29 @@ fast-deep-equal@^3.1.1: resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== +fast-glob@^3.2.7: + version "3.2.7" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1" + integrity sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== +fastq@^1.6.0: + version "1.13.0" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" + integrity sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw== + dependencies: + reusify "^1.0.4" + feedparser@2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/feedparser/-/feedparser-2.2.0.tgz#90279e46af711649a2135307085c9377865adcf3" @@ -748,34 +824,15 @@ fresh@0.5.2: resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= -fs-constants@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" - integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== - fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= -fsevents@~2.1.2: - version "2.1.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e" - integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ== - -gauge@~2.7.3: - version "2.7.4" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" - integrity sha1-LANAXHU4w51+s3sxcCLjJfsBi/c= - dependencies: - aproba "^1.0.3" - console-control-strings "^1.0.0" - has-unicode "^2.0.0" - object-assign "^4.1.0" - signal-exit "^3.0.0" - string-width "^1.0.1" - strip-ansi "^3.0.1" - wide-align "^1.1.0" +fsevents@~2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== getpass@^0.1.1: version "0.1.7" @@ -784,12 +841,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -github-from-package@0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" - integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4= - -glob-parent@~5.1.0: +glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== @@ -820,6 +872,18 @@ glob@~7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" +globby@^12.0.2: + version "12.0.2" + resolved "https://registry.yarnpkg.com/globby/-/globby-12.0.2.tgz#53788b2adf235602ed4cabfea5c70a1139e1ab11" + integrity sha512-lAsmb/5Lww4r7MM9nCCliDZVIKbZTavrsunAsHLr9oHthrZP1qi7/gAnHOsUs9bLvEt2vKVJhHmxuL7QbDuPdQ== + dependencies: + array-union "^3.0.1" + dir-glob "^3.0.1" + fast-glob "^3.2.7" + ignore "^5.1.8" + merge2 "^1.4.1" + slash "^4.0.0" + graceful-cluster@0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/graceful-cluster/-/graceful-cluster-0.0.3.tgz#2c3f57da3610afb28c1bbc8afaa5030dcf506bdc" @@ -848,11 +912,6 @@ has-flag@^3.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= -has-unicode@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" - integrity sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk= - hashring@3.2.x: version "3.2.0" resolved "https://registry.yarnpkg.com/hashring/-/hashring-3.2.0.tgz#fda4efde8aa22cdb97fb1d2a65e88401e1c144ce" @@ -876,17 +935,32 @@ htmlparser2@3.0: domutils "1.1" readable-stream "1.0" -htmlparser2@3.9.2, htmlparser2@^3.9.1: - version "3.9.2" - resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338" - integrity sha1-G9+HrMoPP55T+k/M6w9LTLsAszg= +htmlparser2@^3.9.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f" + integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ== dependencies: - domelementtype "^1.3.0" + domelementtype "^1.3.1" domhandler "^2.3.0" domutils "^1.5.1" entities "^1.1.1" inherits "^2.0.1" - readable-stream "^2.0.2" + readable-stream "^3.1.1" + +htmlparser2@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-7.1.2.tgz#587923d38f03bc89e03076e00cba2c7473f37f7c" + integrity sha512-d6cqsbJba2nRdg8WW2okyD4ceonFHn9jLFxhwlNcLhQWcFPdxXeJulgOLjLKtAK9T6ahd+GQNZwG9fjmGW7lyg== + dependencies: + domelementtype "^2.0.1" + domhandler "^4.2.2" + domutils "^2.8.0" + entities "^3.0.1" + +http-cache-semantics@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" + integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== http-errors@1.7.2, http-errors@~1.7.2: version "1.7.2" @@ -920,10 +994,6 @@ http-errors@~1.8.0: statuses ">= 1.5.0 < 2" toidentifier "1.0.0" -http-parser-js@itteco/http-parser-js#magicode-fix-22: - version "0.4.10" - resolved "https://codeload.github.com/itteco/http-parser-js/tar.gz/18b6b19b41fa3dd5d032fc2ffc58ac909da0a86e" - http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -933,11 +1003,6 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" -iconv-lite@0.4.17: - version "0.4.17" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.17.tgz#4fdaa3b38acbc2c031b045d0edcdfe1ecab18c8d" - integrity sha1-T9qjs4rLwsAxsEXQ7c3+HsqxjI0= - iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" @@ -945,21 +1010,22 @@ iconv-lite@0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" +iconv-lite@^0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + ieee754@^1.1.13: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== -iltorb@^2.4.3: - version "2.4.5" - resolved "https://registry.yarnpkg.com/iltorb/-/iltorb-2.4.5.tgz#d64434b527099125c6839ed48b666247a172ef87" - integrity sha512-EMCMl3LnnNSZJS5QrxyZmMTaAC4+TJkM5woD+xbpm9RB+mFYCr7C05GFE3TEGCsVQSVHmjX+3sf5AiwsylNInQ== - dependencies: - detect-libc "^1.0.3" - nan "^2.14.0" - npmlog "^4.1.2" - prebuild-install "^5.3.3" - which-pm-runs "^1.0.0" +ignore@^5.1.8: + version "5.1.8" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" + integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== inflight@^1.0.4: version "1.0.6" @@ -969,7 +1035,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -979,11 +1045,6 @@ inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= -ini@~1.3.0: - version "1.3.8" - resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" - integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== - ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -1001,17 +1062,10 @@ is-extglob@^2.1.1: resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= -is-fullwidth-code-point@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" - integrity sha1-754xOG8DGn8NZDr4L95QxFfvAMs= - dependencies: - number-is-nan "^1.0.0" - is-glob@^4.0.1, is-glob@~4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" - integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg== + version "4.0.3" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" + integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== dependencies: is-extglob "^2.1.1" @@ -1192,6 +1246,13 @@ lodash@^4.14.0: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +lru-cache@6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + media-typer@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" @@ -1215,11 +1276,24 @@ merge-descriptors@1.0.1: resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= +merge2@^1.3.0, merge2@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" + integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== + methods@^1.1.1, methods@^1.1.2, methods@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= +micromatch@^4.0.4: + version "4.0.4" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9" + integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg== + dependencies: + braces "^3.0.1" + picomatch "^2.2.3" + mime-db@1.44.0: version "1.44.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" @@ -1242,11 +1316,6 @@ mime@1.6.0, mime@^1.4.1: resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== -mimic-response@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43" - integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA== - minimatch@3.0.4, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -1259,21 +1328,11 @@ minimist@0.0.8: resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0= -minimist@^1.2.0, minimist@^1.2.3: - version "1.2.5" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" - integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== - minreq@0.2: version "0.2.3" resolved "https://registry.yarnpkg.com/minreq/-/minreq-0.2.3.tgz#9d7fc1d37f1cccccab55e402c12dbb78c7e3ab0d" integrity sha1-nX/B038czMyrVeQCwS27eMfjqw0= -mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: - version "0.5.3" - resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" - integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== - mkdirp@0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" @@ -1313,15 +1372,15 @@ moment@2.19.3: resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.3.tgz#bdb99d270d6d7fda78cc0fbace855e27fe7da69f" integrity sha1-vbmdJw1tf9p4zA+6zoVeJ/59pp8= -mongodb@3.6.11: - version "3.6.11" - resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.6.11.tgz#8a59a0491a92b00a8c925f72ed9d9a5b054aebb2" - integrity sha512-4Y4lTFHDHZZdgMaHmojtNAlqkvddX2QQBEN0K//GzxhGwlI9tZ9R0vhbjr1Decw+TF7qK0ZLjQT292XgHRRQgw== +mongodb@3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.7.2.tgz#d0d43b08ff1e5c13f4112175e321fa292cf35a3d" + integrity sha512-/Qi0LmOjzIoV66Y2JQkqmIIfFOy7ZKsXnQNlUXPFXChOw3FCdNqVD5zvci9ybm6pkMe/Nw+Rz9I0Zsk2a+05iQ== dependencies: bl "^2.2.1" bson "^1.1.4" denque "^1.4.1" - optional-require "^1.0.3" + optional-require "^1.1.8" safe-buffer "^5.1.2" optionalDependencies: saslprep "^1.0.0" @@ -1332,16 +1391,17 @@ mongoose-legacy-pluralize@1.0.2: integrity sha512-Yo/7qQU4/EyIS8YDFSeenIvXxZN+ld7YdV9LqFVQJzTLye8unujAWPZ4NWKfFA+RNjh+wvTWKY9Z3E5XM6ZZiQ== mongoose@^5.13.7: - version "5.13.7" - resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-5.13.7.tgz#b6938390e93807a6f2ea91a054af2faa67f17218" - integrity sha512-ADIvftZ+KfoTALMZ0n8HvBlezFhcUd73hQaHQDwQ+3X+JZlqE47fUy9yhFZ2SjT+qzmuaCcIXCfhewIc38t2fQ== + version "5.13.12" + resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-5.13.12.tgz#45ad4d8f4b1782cc547e1fa1946608b016d7829f" + integrity sha512-ZEuZ3X/yop9XyOyuCYMz+oxJxXBclm9LIsjKHB0QX2eaNqKNqkvZFzkElbJCj8FDvYmBZFh0OFHlkREhtie6uA== dependencies: + "@types/bson" "1.x || 4.0.x" "@types/mongodb" "^3.5.27" bson "^1.1.4" kareem "2.3.2" - mongodb "3.6.11" + mongodb "3.7.2" mongoose-legacy-pluralize "1.0.2" - mpath "0.8.3" + mpath "0.8.4" mquery "3.2.5" ms "2.1.2" optional-require "1.0.x" @@ -1350,10 +1410,10 @@ mongoose@^5.13.7: sift "13.5.2" sliced "1.0.1" -mpath@0.8.3: - version "0.8.3" - resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.8.3.tgz#828ac0d187f7f42674839d74921970979abbdd8f" - integrity sha512-eb9rRvhDltXVNL6Fxd2zM9D4vKBxjVVQNLNijlj7uoXUy19zNDsIif5zR+pWmPCWNKwAtqyo4JveQm4nfD5+eA== +mpath@0.8.4: + version "0.8.4" + resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.8.4.tgz#6b566d9581621d9e931dd3b142ed3618e7599313" + integrity sha512-DTxNZomBcTWlrMW76jy1wvV37X/cNNxPW1y2Jzd4DZkAaC5ZGsm8bfGfNOthcDuRJujXLqiuS6o3Tpy0JEoh7g== mquery@3.2.5: version "3.2.5" @@ -1395,16 +1455,6 @@ multiparty@^4.1.2: safe-buffer "5.2.1" uid-safe "2.1.5" -nan@^2.14.0: - version "2.14.2" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" - integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== - -napi-build-utils@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" - integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== - negotiator@0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" @@ -1415,13 +1465,6 @@ next-tick@^1.0.0: resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== -node-abi@^2.7.0: - version "2.26.0" - resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.26.0.tgz#355d5d4bc603e856f74197adbf3f5117a396ba40" - integrity sha512-ag/Vos/mXXpWLLAYWsAoQdgS+gW7IwvgMLOgqopm/DbzAjazLltzgzpVMsFlgmo9TzG5hGXeaBZx2AI731RIsQ== - dependencies: - semver "^5.4.1" - node-cache@1.*: version "1.1.0" resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-1.1.0.tgz#186365032d2395bdff73404178fb2bc8981ace70" @@ -1429,11 +1472,6 @@ node-cache@1.*: dependencies: underscore "*" -noop-logger@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2" - integrity sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI= - nopt@~4.0.1: version "4.0.3" resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" @@ -1447,16 +1485,6 @@ normalize-path@^3.0.0, normalize-path@~3.0.0: resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== -npmlog@^4.0.1, npmlog@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" - integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== - dependencies: - are-we-there-yet "~1.1.2" - console-control-strings "~1.1.0" - gauge "~2.7.3" - set-blocking "~2.0.0" - nth-check@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c" @@ -1464,21 +1492,11 @@ nth-check@~1.0.1: dependencies: boolbase "~1.0.0" -number-is-nan@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" - integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= - oauth-sign@~0.9.0: version "0.9.0" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= - on-finished@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" @@ -1486,7 +1504,7 @@ on-finished@~2.3.0: dependencies: ee-first "1.1.1" -once@^1.3.0, once@^1.3.1, once@^1.4.0: +once@^1.3.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= @@ -1498,10 +1516,10 @@ optional-require@1.0.x: resolved "https://registry.yarnpkg.com/optional-require/-/optional-require-1.0.3.tgz#275b8e9df1dc6a17ad155369c2422a440f89cb07" integrity sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA== -optional-require@^1.0.3: - version "1.1.7" - resolved "https://registry.yarnpkg.com/optional-require/-/optional-require-1.1.7.tgz#9ab5b254f59534108d4b2201d9ae96a063abc015" - integrity sha512-cIeRZocXsZnZYn+SevbtSqNlLbeoS4mLzuNn4fvXRMDRNhTGg0sxuKXl0FnZCtnew85LorNxIbZp5OeliILhMw== +optional-require@^1.1.8: + version "1.1.8" + resolved "https://registry.yarnpkg.com/optional-require/-/optional-require-1.1.8.tgz#16364d76261b75d964c482b2406cb824d8ec44b7" + integrity sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA== dependencies: require-at "^1.0.6" @@ -1543,36 +1561,20 @@ path-to-regexp@0.1.7: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w= +path-type@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" + integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== + performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= -picomatch@^2.0.4, picomatch@^2.2.1: - version "2.2.3" - resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.3.tgz#465547f359ccc206d3c48e46a1bcb89bf7ee619d" - integrity sha512-KpELjfwcCDUb9PeigTs2mBJzXUPzAuP2oPcA989He8Rte0+YUAjw1JVedDhuTKPkHjSYzMN3npC9luThGYEKdg== - -prebuild-install@^5.3.3: - version "5.3.6" - resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-5.3.6.tgz#7c225568d864c71d89d07f8796042733a3f54291" - integrity sha512-s8Aai8++QQGi4sSbs/M1Qku62PFK49Jm1CbgXklGz4nmHveDq0wzJkg7Na5QbnO1uNH8K7iqx2EQ/mV0MZEmOg== - dependencies: - detect-libc "^1.0.3" - expand-template "^2.0.3" - github-from-package "0.0.0" - minimist "^1.2.3" - mkdirp-classic "^0.5.3" - napi-build-utils "^1.0.1" - node-abi "^2.7.0" - noop-logger "^0.1.1" - npmlog "^4.0.1" - pump "^3.0.0" - rc "^1.2.7" - simple-get "^3.0.3" - tar-fs "^2.0.0" - tunnel-agent "^0.6.0" - which-pm-runs "^1.0.0" +picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3: + version "2.3.0" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" + integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== probe-image-size@^5.0.0: version "5.0.0" @@ -1608,14 +1610,6 @@ psl@^1.1.28: resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== -pump@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" - integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== - dependencies: - end-of-stream "^1.1.0" - once "^1.3.1" - punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" @@ -1631,6 +1625,11 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +queue-microtask@^1.2.2: + version "1.2.3" + resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" + integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== + random-bytes@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b" @@ -1651,16 +1650,6 @@ raw-body@2.4.0: iconv-lite "0.4.24" unpipe "1.0.0" -rc@^1.2.7: - version "1.2.8" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" - integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== - dependencies: - deep-extend "^0.6.0" - ini "~1.3.0" - minimist "^1.2.0" - strip-json-comments "~2.0.1" - readabilitySAX@1.6.1: version "1.6.1" resolved "https://registry.yarnpkg.com/readabilitySAX/-/readabilitySAX-1.6.1.tgz#4a040cdeeb52c62d7774da0e1b9708b72cebbd00" @@ -1681,7 +1670,7 @@ readable-stream@1.0: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.2.2, readable-stream@^2.3.5: +readable-stream@^2.2.2, readable-stream@^2.3.5: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -1694,7 +1683,7 @@ readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.2.2, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.1.1, readable-stream@^3.4.0: +readable-stream@^3.1.1: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -1716,10 +1705,10 @@ readable-stream@~2.1.5: string_decoder "~0.10.x" util-deprecate "~1.0.1" -readdirp@~3.5.0: - version "3.5.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.5.0.tgz#9ba74c019b15d365278d2e91bb8c48d7b4d42c9e" - integrity sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ== +readdirp@~3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" + integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== dependencies: picomatch "^2.2.1" @@ -1819,17 +1808,29 @@ retry@0.6.0: resolved "https://registry.yarnpkg.com/retry/-/retry-0.6.0.tgz#1c010713279a6fd1e8def28af0c3ff1871caa537" integrity sha1-HAEHEyeab9Ho3vKK8MP/GHHKpTc= +reusify@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" + integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== + +run-parallel@^1.1.9: + version "1.2.0" + resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" + integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== + dependencies: + queue-microtask "^1.2.2" + safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2: +safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -1846,11 +1847,6 @@ sax@1.2.2, sax@^1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.2.tgz#fd8631a23bc7826bef5d871bdb87378c95647828" integrity sha1-/YYxojvHgmvvXYcb24c3jJVkeCg= -semver@^5.4.1: - version "5.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== - semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" @@ -1904,11 +1900,6 @@ serve-static@1.14.1: parseurl "~1.3.3" send "0.17.1" -set-blocking@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" - integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= - setprototypeof@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656" @@ -1929,30 +1920,16 @@ sift@13.5.2: resolved "https://registry.yarnpkg.com/sift/-/sift-13.5.2.tgz#24a715e13c617b086166cd04917d204a591c9da6" integrity sha512-+gxdEOMA2J+AI+fVsCqeNn7Tgx3M9ZN9jdi95939l1IJ8cZsqS8sqpJyOkic2SJk+1+98Uwryt/gL6XDaV+UZA== -signal-exit@^3.0.0: - version "3.0.3" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" - integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== - -simple-concat@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" - integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== - -simple-get@^3.0.3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-3.1.0.tgz#b45be062435e50d159540b576202ceec40b9c6b3" - integrity sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA== - dependencies: - decompress-response "^4.2.0" - once "^1.3.1" - simple-concat "^1.0.0" - simple-lru-cache@0.0.x: version "0.0.2" resolved "https://registry.yarnpkg.com/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz#d59cc3a193c1a5d0320f84ee732f6e4713e511dd" integrity sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0= +slash@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7" + integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew== + sliced@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/sliced/-/sliced-1.0.1.tgz#0b3a662b5d04c3177b1926bea82b03f837a2ef41" @@ -1997,38 +1974,24 @@ stream-parser@~0.3.1: dependencies: debug "2" -string-width@^1.0.1, "string-width@^1.0.2 || 2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" - integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M= - dependencies: - code-point-at "^1.0.0" - is-fullwidth-code-point "^1.0.0" - strip-ansi "^3.0.0" - -string_decoder@^1.1.1, string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== dependencies: - safe-buffer "~5.1.0" + safe-buffer "~5.2.0" string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= -strip-ansi@^3.0.0, strip-ansi@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8= +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== dependencies: - ansi-regex "^2.0.0" - -strip-json-comments@~2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" - integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= + safe-buffer "~5.1.0" superagent@^3.8.3: version "3.8.3" @@ -2068,27 +2031,6 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" -tar-fs@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" - integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== - dependencies: - chownr "^1.1.1" - mkdirp-classic "^0.5.2" - pump "^3.0.0" - tar-stream "^2.1.4" - -tar-stream@^2.1.4: - version "2.2.0" - resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" - integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== - dependencies: - bl "^4.0.3" - end-of-stream "^1.4.1" - fs-constants "^1.0.0" - inherits "^2.0.3" - readable-stream "^3.1.1" - to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" @@ -2192,19 +2134,12 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" -which-pm-runs@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" - integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs= - -wide-align@^1.1.0: - version "1.1.3" - resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457" - integrity sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA== - dependencies: - string-width "^1.0.2 || 2" - wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==