From 5d27d1f5f7b930df686e45f054855864c276d69e Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 13 Oct 2021 17:07:08 +0300
Subject: [PATCH 001/102] node-fetch: update using new response object

---
 lib/agent-http2.js                          |   2 +
 lib/core.js                                 |   7 +-
 lib/plugins/system/htmlparser/htmlparser.js |  59 +++++-----
 lib/plugins/system/meta/meta.js             |   2 +-
 lib/plugins/system/oembed/oembedUtils.js    |  12 +-
 lib/plugins/system/readability.js           |   2 -
 lib/utils.js                                | 115 ++++++++++++--------
 package.json                                |   2 +
 plugins/custom/http-headers.js              |   4 +-
 plugins/custom/noindex/noindex-header.js    |   2 +-
 yarn.lock                                   |  37 +++++++
 11 files changed, 158 insertions(+), 86 deletions(-)

diff --git a/lib/agent-http2.js b/lib/agent-http2.js
index 027b0856c..3fb7ada1a 100644
--- a/lib/agent-http2.js
+++ b/lib/agent-http2.js
@@ -38,7 +38,9 @@ function Http2Response(clientHttp2Stream) {
 
     clientHttp2Stream.on('response', (headers) => {
 
+        // TODO: status?
         that.statusCode = headers[':status'];
+        // TODO: headers wrapper?
         that.headers = headers;
 
         that.emit('ready');
diff --git a/lib/core.js b/lib/core.js
index e102fd26e..e206c1b72 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -878,7 +878,8 @@
 
                                 // Debug http2 state.
     
-                                allResults.h2 = r.data.htmlparser.request.response.h2;
+                                // TODO: detect h2
+                                //allResults.h2 = r.data.htmlparser.request.response.h2;
                             }
 
                             // First data has priority, do not override it.
@@ -1501,8 +1502,8 @@
             aborted = false,
 
             abortCurrentRequest = function() {
-                if (context.htmlparser) {
-                    utils.abortRequest(context.htmlparser.request);
+                if (context.htmlparser && context.htmlparser.abortController) {
+                    context.htmlparser.abortController.abort();
                 }
             },
 
diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index eb774da80..18f9fd0f3 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -21,7 +21,7 @@ module.exports = {
 
     getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, cb) {
 
-        var request;
+        var abortController;
 
         getUrlFunctional(url, _.extend({}, options, {
             followRedirect: !!options.followHTTPRedirect
@@ -37,19 +37,19 @@ module.exports = {
                     });
                 }
             },
-            onRequest: function(req) {
-                request = req;
+            onAbortController: function(controller) {
+                abortController = controller;
             },
             onResponse: function(resp) {
                 function cacheAndRespond(error, data, preventCache) {
 
                     var skip_cache = preventCache;
 
-                    if (CONFIG.TEMP_HTTP_ERROR_CODES.indexOf(resp.statusCode) > -1) {
+                    if (CONFIG.TEMP_HTTP_ERROR_CODES.indexOf(resp.status) > -1) {
                         // Skip caching TEMP_HTTP_ERROR_CODES.
                         skip_cache = true;
                     }
-                    if (resp.statusCode >= 500 && resp.statusCode <= 599) {
+                    if (resp.status >= 500 && resp.status <= 599) {
                         // Skip caching 5xx errors.
                         skip_cache = true;
                     }
@@ -88,43 +88,45 @@ module.exports = {
                     }
                 }
 
-                if (resp.statusCode >= 300 && resp.statusCode < 400 && resp.headers.location) {
-                    utils.abortRequest(request);
-                    var redirectUrl = urlLib.resolve(url, resp.headers.location);
+                var headers = resp.headers.raw();
+
+                if (resp.status >= 300 && resp.status< 400 && headers.location) {
+                    abortController.abort();
+                    var redirectUrl = urlLib.resolve(url, headers.location);
                     // Prevent cache self redirect. Some sites changes cookies and stops redirect loop (e.g. https://miro.com/app/live-embed/o9J_lBwNMhI=/?embedAutoplay=true)
                     var preventCache = redirectUrl === url;
                     if (!options.exposeStatusCode) {
                         return cacheAndRespond({
-                            redirect: resp.headers.location
+                            redirect: headers.location
                         }, null, preventCache);
                     } else {
                         return cacheAndRespond(null, {
-                            headers: resp.headers
+                            headers: headers
                         });
                     }
                 }
 
-                if (resp.statusCode !== 200) {
-                    utils.abortRequest(request);
+                if (resp.status !== 200) {
+                    abortController.abort();
                     if (!!options.exposeStatusCode) {
                         return cacheAndRespond(null, {
-                            __statusCode: resp.statusCode,
-                            headers: resp.headers
+                            __statusCode: resp.status,
+                            headers: headers
                         });
                     } else {
                         return cacheAndRespond({
-                            responseStatusCode: resp.statusCode
+                            responseStatusCode: resp.status
                         });
                     }
                 }
 
-                if('content-type' in resp.headers && !/text\/html|application\/xhtml\+xml/gi.test(resp.headers['content-type'])){
-                    utils.abortRequest(request);
+                if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){
+                    abortController.abort();
                     return cacheAndRespond(null, {
                         __nonHtmlContentData: {
-                            type: resp.headers['content-type'],
-                            content_length: resp.headers['content-length'],
-                            'set-cookie': resp.headers['set-cookie']
+                            type: headers['content-type'],
+                            content_length: headers['content-length'],
+                            'set-cookie': headers['set-cookie']
                         }
                     });
                 }
@@ -132,19 +134,22 @@ module.exports = {
                 // Init htmlparser handler.
                 var handler = new CollectingHandlerForMutliTarget();
                 handler.onNoHandlers = function() {
-                    if (!request.response.isPaused()) {
-                        request.response.pause();
-                    }
+                    // TODO:
+                    // if (!request.response.isPaused()) {
+                    //     request.response.pause();
+                    // }
                 };
                 handler.onFirstHandler = function() {
-                    if (request.response.isPaused()) {
-                        request.response.resume();
-                    }
+                    // TODO:
+                    // if (request.response.isPaused()) {
+                    //     request.response.resume();
+                    // }
                 };
                 var parser = new Parser(handler, {
                     lowerCaseTags: true
                 });
-                handler.request = request;
+                handler.headers = headers;
+                handler.abortController = abortController;
 
                 // Do before resume?
                 cb(null, {
diff --git a/lib/plugins/system/meta/meta.js b/lib/plugins/system/meta/meta.js
index 1a7c3a457..ddf76704f 100644
--- a/lib/plugins/system/meta/meta.js
+++ b/lib/plugins/system/meta/meta.js
@@ -13,7 +13,7 @@ module.exports = {
 
         var metaHandler = new HTMLMetaHandler(
             url,
-            htmlparser.request.response.headers["content-type"],
+            htmlparser.headers["content-type"],
             function(error, meta) {
 
                 htmlparser.removeHandler(metaHandler);
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index 1c5030d3b..ee9b1389e 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -232,12 +232,12 @@ module.exports.getOembed = function(uri, options, callback) {
             }, {
                 onError: cbWrapper,
                 onResponse: function(res) {
-                    if (res.statusCode == 200) {
+                    if (res.status == 200) {
                         parseResponse(res);
                     } else if (options && options.parseErrorBody) {
-                        parseResponse(res, res.statusCode);
+                        parseResponse(res, res.status);
                     } else {
-                        cbWrapper(res.statusCode);
+                        cbWrapper(res.status);
                     }
                 }
             });
@@ -266,7 +266,7 @@ module.exports.getOembed = function(uri, options, callback) {
  * Convert XML or JSON stream to an oEmbed object.
  */
 function stream2oembed(uri, stream, callback) {
-    var contentType = stream.headers && stream.headers['content-type'];
+    var contentType = stream.headers && stream.headers.get('content-type');
     if (contentType && contentType.match('xml')) {
 
         xmlStream2oembed(stream, callback)
@@ -308,7 +308,7 @@ function xmlStream2oembed(stream, callback) {
     var value;
     var firstTag;
 
-    var charset = getCharset(stream.headers && stream.headers['content-type']);
+    var charset = getCharset(stream.headers && stream.headers.get('content-type'));
 
     var saxStream = sax.createStream();
 
@@ -382,7 +382,7 @@ function xmlStream2oembed(stream, callback) {
  */
 function jsonStream2oembed(stream, callback) {
 
-    var charset = getCharset(stream.headers && stream.headers['content-type']);
+    var charset = getCharset(stream.headers && stream.headers.get('content-type'));
 
     var data = "";
     stream.on('data', function(chunk) {
diff --git a/lib/plugins/system/readability.js b/lib/plugins/system/readability.js
index 226744e86..989ce0da8 100644
--- a/lib/plugins/system/readability.js
+++ b/lib/plugins/system/readability.js
@@ -44,8 +44,6 @@ module.exports = {
         readability.onend = onEnd;
 
         htmlparser.addHandler(readability);
-
-        htmlparser.request.response.on('onerror', cb);
     }
 
 };
\ No newline at end of file
diff --git a/lib/utils.js b/lib/utils.js
index 00be2f252..a7907b5e6 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,5 +1,7 @@
 var events = require('events');
 var request = require('request');
+var fetch = require('node-fetch');
+const AbortController = require('abort-controller');
 var Http2Agent = require('./agent-http2').Http2Agent;
 var zlib = require('zlib');
 var decompressStream = require('iltorb').decompressStream;
@@ -118,7 +120,9 @@ var prepareRequestOptions = exports.prepareRequestOptions = function(request_opt
             || request_options.method === 'GET'
             || request_options.method === 'HEAD')) {
         // http2 only for ssl and without proxy.
-        request_options.agent = http2Agent;
+        
+        // TODO:
+        // request_options.agent = http2Agent;
     }
 
     var lang = options && options.getProviderOptions && options.getProviderOptions('locale', 'en-US');
@@ -154,8 +158,8 @@ var getUrlFunctional = exports.getUrlFunctional = function(url, options, callbac
 
             callbacks.onError && callbacks.onError(error);
         })
-        .on('request', function(request) {
-            callbacks.onRequest && callbacks.onRequest(request);
+        .on('abortController', function(abortController) {
+            callbacks.onAbortController && callbacks.onAbortController(abortController);
         })
         .on('response', function(response) {
             callbacks.onResponse && callbacks.onResponse(response);
@@ -174,6 +178,8 @@ var getUrlFunctional = exports.getUrlFunctional = function(url, options, callbac
  */
 var getUrl = exports.getUrl = function(url, options) {
 
+    const controller = new AbortController();
+
     var req = new events.EventEmitter();
 
     var options = options || {};
@@ -189,6 +195,7 @@ var getUrl = exports.getUrl = function(url, options) {
 
             var sslProtocol = /^(https:)?\/\//i.test(url);
 
+            // TODO: review node-fetch options
             var request_options = prepareRequestOptions({
                 uri: url,
                 method: 'GET',
@@ -201,16 +208,16 @@ var getUrl = exports.getUrl = function(url, options) {
                 maxRedirects: options.maxRedirects || CONFIG.MAX_REDIRECTS,
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
                 followRedirect: options.followRedirect,
-                jar: jar
+                jar: jar,
+                signal: controller.signal
             }, options);
 
-            var r = request(request_options)
-                .on('error', function(error) {
-                    req.emit('error', error, {http2Agent: request_options.agent === http2Agent});
-                })
-                .on('response', function(res) {
+            fetch(request_options.url || request_options.uri, request_options)
+                .then(res => {
+
+                    var headers = res.headers.raw();
 
-                    var contentEncoding = res.headers['content-encoding'];
+                    var contentEncoding = headers['content-encoding'];
                     contentEncoding = contentEncoding && contentEncoding.trim().toLowerCase();
 
                     var zlibOptions = {
@@ -222,14 +229,17 @@ var getUrl = exports.getUrl = function(url, options) {
 
                         var br = decompressStream();
 
+                        // TODO:
                         br.request = res.request;
-                        br.statusCode = res.statusCode;
-                        br.headers = res.headers;
+                        // TODO:
+                        br.statusCode = res.status;
+                        br.headers = headers;
 
                         if (!options.asBuffer) {
                             br.setEncoding("binary");
                         }
 
+                        // TODO:
                         req.emit('response', br);
                         res.pipe(br);
 
@@ -237,14 +247,17 @@ var getUrl = exports.getUrl = function(url, options) {
 
                         var gunzip = contentEncoding === 'gzip' ? zlib.createGunzip(zlibOptions) : zlib.createInflate(zlibOptions);
 
+                        // TODO:
                         gunzip.request = res.request;
-                        gunzip.statusCode = res.statusCode;
-                        gunzip.headers = res.headers;
+                        // TODO:
+                        gunzip.statusCode = res.status;
+                        gunzip.headers = headers;
 
                         if (!options.asBuffer) {
                             gunzip.setEncoding("binary");
                         }
 
+                        // TODO:
                         req.emit('response', gunzip);
                         res.pipe(gunzip);
 
@@ -254,11 +267,16 @@ var getUrl = exports.getUrl = function(url, options) {
                             res.setEncoding("binary");
                         }
 
+                        // TODO:
                         req.emit('response', res);
                     }
+                })
+                .catch(error => {
+                    req.emit('error', error, {http2Agent: request_options.agent === http2Agent});
                 });
 
-            req.emit('request', r);
+
+            req.emit('abortController', controller);
 
         } catch (ex) {
             console.error('Error on getUrl for', url, '.\n Error:' + ex);
@@ -286,8 +304,8 @@ var getHeadFunctional = exports.getHeadFunctional = function(url, options, callb
 
             callbacks.onError && callbacks.onError(error);
         })
-        .on('request', function(request) {
-            callbacks.onRequest && callbacks.onRequest(request);
+        .on('abortController', function(abortController) {
+            callbacks.onAbortController && callbacks.onAbortController(abortController);
         })
         .on('response', function(response) {
             callbacks.onResponse && callbacks.onResponse(response);
@@ -296,6 +314,8 @@ var getHeadFunctional = exports.getHeadFunctional = function(url, options, callb
 
 var getHead = function(url, options) {
 
+    const controller = new AbortController();
+
     var req = new events.EventEmitter();
 
     var options = options || {};
@@ -308,7 +328,9 @@ var getHead = function(url, options) {
 
     process.nextTick(function() {
         try {
-            var r = request(prepareRequestOptions({
+
+            // TODO: review node-fetch options
+            var request_options = prepareRequestOptions({
                 uri: url,
                 method: 'HEAD',
                 headers: {
@@ -322,18 +344,21 @@ var getHead = function(url, options) {
                 agentOptions: {
                     // getHead called for video, need check certificate.
                     rejectUnauthorized: true
-                }
+                },
+                signal: controller.signal
             }, {
                 disableHttp2: options && options.disableHttp2
-            }))
-                .on('error', function(error) {
-                    req.emit('error', error);
-                })
-                .on('response', function(res) {
+            });
+
+            fetch(request_options.url || request_options.uri, request_options)
+                .then(res => {
                     req.emit('response', res);
+                })
+                .catch(error => {
+                    req.emit('error', error);
                 });
 
-            req.emit('request', r);
+            req.emit('abortController', controller);
 
         } catch (ex) {
             console.error('Error on getHead for', url, '.\n Error:' + ex);
@@ -478,7 +503,7 @@ exports.getImageMetadata = function(uri, options, callback){
     cache.withCache("image-meta:" + uri, function(callback) {
 
         var loadImageHead, imageResponseStarted, totalTime, timeout, contentLength;
-        var requestInstance = null;
+        var abortController;
 
         function finish(error, data) {
 
@@ -490,7 +515,7 @@ exports.getImageMetadata = function(uri, options, callback){
             }
 
             // We don't need more data. Abort causes error. timeout === null here so error will be skipped.
-            abortRequest(requestInstance)
+            abortController.abort();
 
             if (!error && !data) {
                 error = 404;
@@ -563,25 +588,25 @@ exports.getImageMetadata = function(uri, options, callback){
                     asBuffer: true,
                     disableHttp2: options && options.disableHttp2
                 }, {
-                    onRequest: function(req) {
-                        requestInstance = req;
+                    onAbortController: function(controller) {
+                        abortController = controller;
                     },
                     onResponse: function(res) {
 
-                        var content_type = res.headers['content-type'];
+                        var content_type = res.headers.get('content-type');
 
                         if (content_type && content_type !== 'application/octet-stream' && content_type !== 'binary/octet-stream') {
 
                             if (content_type.indexOf('image') === -1 && !uri.match(/\.(jpg|png|gif|webp)(\?.*)?$/i)) {
-                                return finishCb('invalid content type: ' + res.headers['content-type'], undefined, 'onResponse content_type');
+                                return finishCb('invalid content type: ' + content_type, undefined, 'onResponse content_type');
                             }
                         }
 
-                        if (res.statusCode == 200) {
+                        if (res.status == 200) {
                             if (options.debug) {
                                 imageResponseStarted = totalTime();
                             }
-                            contentLength = parseInt(res.headers['content-length'] || '0', 10);
+                            contentLength = parseInt(res.headers.get('content-length') || '0', 10);
                             imagesize(res, function(error, data) {
                                 if (data && data.type) {
                                     data.format = data.type;
@@ -589,7 +614,7 @@ exports.getImageMetadata = function(uri, options, callback){
                                 finishCb(error, data, 'imagesize');
                             });
                         } else {
-                            finishCb(res.statusCode, undefined, 'onResponse !200');
+                            finishCb(res.status, undefined, 'onResponse !200');
                         }
                     },
                     onError: function(error) {
@@ -679,7 +704,7 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) {
 
     cache.withCache("content-type:" + uriForCache, function(cb) {
 
-        var timeout, requestInstance, totalTime;
+        var timeout, totalTime, abortController;
 
         function finish(error, headers) {
 
@@ -691,7 +716,7 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) {
             }
 
             // We don't need more data. Abort causes error. timeout === null here so error will be skipped.
-            abortRequest(requestInstance);
+            abortController.abort();
 
             var data = {};
 
@@ -745,11 +770,11 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) {
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
                 disableHttp2: options && options.disableHttp2
             }, {
-                onRequest: function(req) {
-                    requestInstance = req;                    
+                onAbortController: function(controller) {
+                    abortController = controller;
                 },
                 onResponse: function(res) {
-                    var error = res.statusCode && res.statusCode != 200 ? res.statusCode : null;
+                    var error = res.status && res.status != 200 ? res.status : null;
 
                     if (!method
                         // If method HEAD is not allowed. ex. Amazon S3 (=403)
@@ -759,17 +784,19 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) {
                             || error === 400 
                             || error >= 500
                             // Or ClourFront that gobbles up headers when checking CORS.
-                            || (res.headers && !res.headers['access-control-allow-origin']
-                                && res.headers.server === 'AmazonS3' && !error ))) {
+                            || (res.headers && !res.headers.get('access-control-allow-origin')
+                                && res.headers.get('server') === 'AmazonS3' && !error ))) {
                         makeCall('GET');
                         return;
                     }
 
+                    // TODO: what is res.request.href ?
+                    // TODO: where used res.headers.location ?
                     if (res.request && res.request.href && res.headers && res.request.href !== uriOriginal) {
-                        res.headers.location = res.request.href;
+                        //res.headers.location = res.request.href;
                     }
 
-                    finish(error, res.headers);
+                    finish(error, res.headers.raw());
                 },
                 onError: function(error) {
                     finish(error);
@@ -1130,7 +1157,7 @@ var getUriStatus = function(uri, options, cb) {
                     content_length: res.headers && res.headers['content-length'] ? parseInt(res.headers['content-length'] || '0', 10) : null
                 };
                 if (options.checkHeaders) {
-                    data.headers = res.headers;
+                    data.headers = res.headers.raw();
                 }
                 cb(null, data);
             });
diff --git a/package.json b/package.json
index 11cef840c..b8084673f 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
     },
     "license": "MIT",
     "dependencies": {
+        "abort-controller": "^3.0.0",
         "async": "2.4.1",
         "cheerio": "0.22.0",
         "chokidar": "^3.3.1",
@@ -35,6 +36,7 @@
         "memcached": "2.2.2",
         "moment": "2.19.3",
         "node-cache": "1.*",
+        "node-fetch": "^3.0.0",
         "parse-iso-duration": "1.0.0",
         "probe-image-size": "^5.0.0",
         "readabilitySAX": "1.6.1",
diff --git a/plugins/custom/http-headers.js b/plugins/custom/http-headers.js
index c3de6bbf5..c5fdebc8a 100644
--- a/plugins/custom/http-headers.js
+++ b/plugins/custom/http-headers.js
@@ -4,9 +4,9 @@ module.exports = {
 
     getData: function(htmlparser, cb) {
 
-        if (htmlparser.request && htmlparser.request.response && htmlparser.request.response.headers) {
+        if (htmlparser.headers) {
             return cb (null, {
-                headers: htmlparser.request.response.headers
+                headers: htmlparser.headers
             })
         } else {
             cb();
diff --git a/plugins/custom/noindex/noindex-header.js b/plugins/custom/noindex/noindex-header.js
index 4c91f1c83..bab65f360 100644
--- a/plugins/custom/noindex/noindex-header.js
+++ b/plugins/custom/noindex/noindex-header.js
@@ -3,7 +3,7 @@ var pluginUtils = require('./utils');
 module.exports = {
 
     getData: function(htmlparser, cb) {
-        var headers = htmlparser.request.response.headers;
+        var headers = htmlparser.headers;
         if (pluginUtils.checkRobots(headers['x-robots-tag'], cb)) {
             return;
         } else {
diff --git a/yarn.lock b/yarn.lock
index d169cc85f..4ddf3e494 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -27,6 +27,13 @@ abbrev@1:
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
   integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
 
+abort-controller@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
+  integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
+  dependencies:
+    event-target-shim "^5.0.0"
+
 accepts@~1.3.7:
   version "1.3.7"
   resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd"
@@ -428,6 +435,11 @@ dashdash@^1.12.0:
   dependencies:
     assert-plus "^1.0.0"
 
+data-uri-to-buffer@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz#594b8973938c5bc2c33046535785341abc4f3636"
+  integrity sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==
+
 debug@2, debug@2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -612,6 +624,11 @@ etag@~1.8.1:
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
   integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
 
+event-target-shim@^5.0.0:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
+  integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
+
 exit@~0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c"
@@ -692,6 +709,13 @@ feedparser@2.2.0:
     readable-stream "^2.2.2"
     sax "^1.2.1"
 
+fetch-blob@^3.1.2:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.1.2.tgz#6bc438675f3851ecea51758ac91f6a1cd1bacabd"
+  integrity sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==
+  dependencies:
+    web-streams-polyfill "^3.0.3"
+
 filelist@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.2.tgz#80202f21462d4d1c2e214119b1807c1bc0380e5b"
@@ -1429,6 +1453,14 @@ node-cache@1.*:
   dependencies:
     underscore "*"
 
+node-fetch@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.0.0.tgz#79da7146a520036f2c5f644e4a26095f17e411ea"
+  integrity sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==
+  dependencies:
+    data-uri-to-buffer "^3.0.1"
+    fetch-blob "^3.1.2"
+
 noop-logger@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2"
@@ -2192,6 +2224,11 @@ verror@1.10.0:
     core-util-is "1.0.2"
     extsprintf "^1.2.0"
 
+web-streams-polyfill@^3.0.3:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz#1516f2d4ea8f1bdbfed15eb65cb2df87098c8364"
+  integrity sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q==
+
 which-pm-runs@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb"

From 8b2c74c68de2b6dd61f2b62090d02aab04ff477d Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 13 Oct 2021 22:46:24 +0300
Subject: [PATCH 002/102] plugins: simple export

---
 plugins/custom/http-headers.js                             | 2 +-
 plugins/custom/oembed-error.js                             | 2 +-
 plugins/custom/oembed-iframe.js                            | 2 +-
 plugins/custom/twitter-image-rel-image.js                  | 2 +-
 plugins/domains/500px/500px.com.js                         | 2 +-
 plugins/domains/absnews.go.com.js                          | 2 +-
 plugins/domains/animoto.com.js                             | 2 +-
 plugins/domains/bandcamp.com.js                            | 2 +-
 plugins/domains/beatport.com.js                            | 2 +-
 plugins/domains/bigthink.com.js                            | 2 +-
 plugins/domains/box.com.js                                 | 2 +-
 plugins/domains/brainyquote.com.js                         | 2 +-
 plugins/domains/buzzfeed.com/buzzfeed.isvideo.js           | 2 +-
 plugins/domains/buzzfeed.com/buzzfeed.video.js             | 2 +-
 plugins/domains/c-span.org/c-span.org.js                   | 2 +-
 plugins/domains/cbc.ca.js                                  | 2 +-
 plugins/domains/channel9.msdn.com.js                       | 2 +-
 plugins/domains/cnevids.com.js                             | 2 +-
 plugins/domains/cnn.com.js                                 | 2 +-
 plugins/domains/d.pr.js                                    | 2 +-
 plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js | 2 +-
 plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js  | 2 +-
 plugins/domains/dailymail.co.uk/mol.im.js                  | 2 +-
 plugins/domains/dailymotion.com/dailymotion.playlist.js    | 2 +-
 plugins/domains/dailymotion.com/dailymotion.swf.js         | 2 +-
 plugins/domains/documentcloud.org.js                       | 2 +-
 plugins/domains/dribbble.com.js                            | 2 +-
 plugins/domains/ebaumsworld.com.js                         | 2 +-
 plugins/domains/espn.com.js                                | 2 +-
 plugins/domains/facebook.com/facebook.page.js              | 2 +-
 plugins/domains/facebook.com/facebook.post.js              | 2 +-
 plugins/domains/facebook.com/facebook.video.js             | 2 +-
 plugins/domains/flickr.com/flickr.gallery.js               | 2 +-
 plugins/domains/flickr.com/flickr.photo.js                 | 2 +-
 plugins/domains/geogebra.iframe.js                         | 2 +-
 plugins/domains/geogebra.org.js                            | 2 +-
 plugins/domains/giphy.com.js                               | 2 +-
 plugins/domains/github.gist.js                             | 2 +-
 plugins/domains/google.com/accounts.google.com.js          | 2 +-
 plugins/domains/google.com/google.calendar.js              | 2 +-
 plugins/domains/google.com/google.custommap.js             | 2 +-
 plugins/domains/google.com/google.maps.js                  | 2 +-
 plugins/domains/google.com/google.streetview.js            | 2 +-
 plugins/domains/hockeydb.com.js                            | 2 +-
 plugins/domains/i.gfycat.com.js                            | 2 +-
 plugins/domains/i.gifs.com.js                              | 2 +-
 plugins/domains/i.giphy.com.js                             | 2 +-
 plugins/domains/ignore.js                                  | 2 +-
 plugins/domains/imageshack.com.js                          | 2 +-
 plugins/domains/imdb.com.js                                | 2 +-
 plugins/domains/issuu.com.js                               | 2 +-
 plugins/domains/jsfiddle.net.js                            | 2 +-
 plugins/domains/knightlab.js                               | 2 +-
 plugins/domains/libsyn.com.js                              | 2 +-
 plugins/domains/live.amcharts.com.js                       | 2 +-
 plugins/domains/livestream.com.js                          | 2 +-
 plugins/domains/mail.ru.js                                 | 2 +-
 plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js     | 2 +-
 plugins/domains/medium.com.js                              | 2 +-
 plugins/domains/momindum.com.js                            | 2 +-
 plugins/domains/nbcnews.com.js                             | 2 +-
 plugins/domains/nbcsports.com.js                           | 2 +-
 plugins/domains/nhl.com/nhl.com.js                         | 2 +-
 plugins/domains/npr.org/npr.sections.js                    | 2 +-
 plugins/domains/nytimes.video.js                           | 2 +-
 plugins/domains/ow.ly.js                                   | 2 +-
 plugins/domains/padlet.com.js                              | 2 +-
 plugins/domains/pastebin.com.js                            | 2 +-
 plugins/domains/piktochart.com/piktochart.com.js           | 2 +-
 plugins/domains/pinterest.com/pinterest.board.js           | 2 +-
 plugins/domains/pinterest.com/pinterest.pin.js             | 2 +-
 plugins/domains/pinterest.com/pinterest.user.js            | 2 +-
 plugins/domains/polldaddy.com.js                           | 2 +-
 plugins/domains/prezi.com.js                               | 2 +-
 plugins/domains/quizlet.com.js                             | 2 +-
 plugins/domains/rapgenius.com.js                           | 2 +-
 plugins/domains/readymag.com.js                            | 2 +-
 plugins/domains/simplecast.com.js                          | 2 +-
 plugins/domains/slid.es.js                                 | 2 +-
 plugins/domains/smugmug.com.js                             | 2 +-
 plugins/domains/soundcloud.com/soundcloud-fallbacks.js     | 2 +-
 plugins/domains/soundcloud.com/soundcloud-oembed-error.js  | 2 +-
 plugins/domains/spotify.com.js                             | 2 +-
 plugins/domains/sverigesradio.se.js                        | 2 +-
 plugins/domains/ted.com/ted.playlists.js                   | 2 +-
 plugins/domains/test.com/test1.js                          | 2 +-
 plugins/domains/test.com/test2.js                          | 2 +-
 plugins/domains/test.com/test3.js                          | 2 +-
 plugins/domains/theguardian.com/guardian.video.js          | 2 +-
 plugins/domains/theta360.com/theta360.com.js               | 2 +-
 plugins/domains/tmz.com.js                                 | 2 +-
 plugins/domains/twitch.tv/clips.twitch.tv.js               | 2 +-
 plugins/domains/twitch.tv/twitch-live-fallback.js          | 2 +-
 plugins/domains/twitter.com/twitter.timelines.js           | 2 +-
 plugins/domains/usatoday.com.js                            | 2 +-
 plugins/domains/vbox7.com.js                               | 2 +-
 plugins/domains/vimeo-moogaloop.js                         | 2 +-
 plugins/domains/washingtonpost.com.js                      | 2 +-
 plugins/domains/widgetic.com.js                            | 2 +-
 plugins/domains/wistia.com.js                              | 2 +-
 plugins/domains/youku.com.js                               | 2 +-
 plugins/domains/youtube.com/youtube.shared.js              | 2 +-
 plugins/links/article/check-article.js                     | 2 +-
 plugins/links/canonical-redirect.js                        | 2 +-
 plugins/links/embedURL/checkEmbedURL.js                    | 2 +-
 plugins/links/favicon.js                                   | 2 +-
 plugins/links/hosted/brightcove-allow-in-page.js           | 2 +-
 plugins/links/hosted/brightcove-hosted.js                  | 2 +-
 plugins/links/hosted/brightcove-in-page-promo.js           | 2 +-
 plugins/links/hosted/cloudapp-hosted.js                    | 2 +-
 plugins/links/hosted/cnevids-hosted.js                     | 2 +-
 plugins/links/hosted/hosted-via-canonical.js               | 2 +-
 plugins/links/hosted/known-players.js                      | 2 +-
 plugins/links/hosted/libsyn-hosted.js                      | 2 +-
 plugins/links/hosted/podbean-hosted.js                     | 2 +-
 plugins/links/hosted/promo-link.js                         | 2 +-
 plugins/links/hosted/tout-hosted.js                        | 2 +-
 plugins/links/hosted/video_src-hosted.js                   | 2 +-
 plugins/links/logo.js                                      | 2 +-
 plugins/links/ms-doc-viewer.js                             | 2 +-
 plugins/links/oembed-icon.js                               | 2 +-
 plugins/links/oembed-photo.js                              | 2 +-
 plugins/links/oembed-rich.js                               | 2 +-
 plugins/links/oembed-thumbnail.js                          | 2 +-
 plugins/links/parsely-image.js                             | 2 +-
 plugins/links/prerender/utils.js                           | 2 +-
 plugins/links/twitter-image.js                             | 2 +-
 plugins/links/twitter-player.js                            | 2 +-
 plugins/links/video_src.js                                 | 2 +-
 plugins/meta/amphtml.js                                    | 2 +-
 plugins/meta/author.js                                     | 2 +-
 plugins/meta/canonical.js                                  | 2 +-
 plugins/meta/category.js                                   | 2 +-
 plugins/meta/copyright.js                                  | 2 +-
 plugins/meta/date.js                                       | 2 +-
 plugins/meta/dc-title.js                                   | 2 +-
 plugins/meta/dc.js                                         | 2 +-
 plugins/meta/description.js                                | 2 +-
 plugins/meta/embedurl-meta.js                              | 2 +-
 plugins/meta/geo-url.js                                    | 2 +-
 plugins/meta/html-title.js                                 | 2 +-
 plugins/meta/keywords.js                                   | 2 +-
 plugins/meta/ld-article-keywords.js                        | 2 +-
 plugins/meta/meta-music.js                                 | 2 +-
 plugins/meta/meta-title.js                                 | 2 +-
 plugins/meta/noscript-redirect.js                          | 2 +-
 plugins/meta/oembed-author.js                              | 2 +-
 plugins/meta/oembed-canonical.js                           | 2 +-
 plugins/meta/oembed-duration.js                            | 2 +-
 plugins/meta/oembed-license.js                             | 2 +-
 plugins/meta/oembed-product.js                             | 2 +-
 plugins/meta/oembed-site.js                                | 2 +-
 plugins/meta/oembed-title.js                               | 2 +-
 plugins/meta/og-article.js                                 | 2 +-
 plugins/meta/og-date.js                                    | 2 +-
 plugins/meta/og-description.js                             | 2 +-
 plugins/meta/og-location.js                                | 2 +-
 plugins/meta/og-product.js                                 | 2 +-
 plugins/meta/og-site.js                                    | 2 +-
 plugins/meta/og-title.js                                   | 2 +-
 plugins/meta/og-video-duration.js                          | 2 +-
 plugins/meta/parsely.js                                    | 2 +-
 plugins/meta/shortlink.js                                  | 2 +-
 plugins/meta/site.js                                       | 2 +-
 plugins/meta/theme-color.js                                | 2 +-
 plugins/meta/twitter-description.js                        | 2 +-
 plugins/meta/twitter-labels.js                             | 2 +-
 plugins/meta/twitter-site.js                               | 2 +-
 plugins/meta/twitter-title.js                              | 2 +-
 plugins/meta/video-duration.js                             | 2 +-
 plugins/meta/video.js                                      | 2 +-
 plugins/meta/yandex-ovs.js                                 | 2 +-
 172 files changed, 172 insertions(+), 172 deletions(-)

diff --git a/plugins/custom/http-headers.js b/plugins/custom/http-headers.js
index c5fdebc8a..2e8a7732c 100644
--- a/plugins/custom/http-headers.js
+++ b/plugins/custom/http-headers.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: "headers",
 
diff --git a/plugins/custom/oembed-error.js b/plugins/custom/oembed-error.js
index a320c5063..b0a88bffd 100644
--- a/plugins/custom/oembed-error.js
+++ b/plugins/custom/oembed-error.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getData: function(oembedError, cb) {
 
diff --git a/plugins/custom/oembed-iframe.js b/plugins/custom/oembed-iframe.js
index 464093447..2e997eec5 100644
--- a/plugins/custom/oembed-iframe.js
+++ b/plugins/custom/oembed-iframe.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: "iframe",
 
diff --git a/plugins/custom/twitter-image-rel-image.js b/plugins/custom/twitter-image-rel-image.js
index 828df579f..d6c122372 100644
--- a/plugins/custom/twitter-image-rel-image.js
+++ b/plugins/custom/twitter-image-rel-image.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(twitter) {
 
diff --git a/plugins/domains/500px/500px.com.js b/plugins/domains/500px/500px.com.js
index bc2a01a76..d0a05aeb5 100644
--- a/plugins/domains/500px/500px.com.js
+++ b/plugins/domains/500px/500px.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:web\.)?500px\.com\/photo\/(\d+)/i,
 
diff --git a/plugins/domains/absnews.go.com.js b/plugins/domains/absnews.go.com.js
index c575b940c..6a876ff6e 100644
--- a/plugins/domains/absnews.go.com.js
+++ b/plugins/domains/absnews.go.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/abcnews\.go\.com\/\w+\/(?:\w+\/)?video\/[a-zA-Z0-9\-_]+\-(\d+)/i        
diff --git a/plugins/domains/animoto.com.js b/plugins/domains/animoto.com.js
index bf983ce7b..0bc060041 100644
--- a/plugins/domains/animoto.com.js
+++ b/plugins/domains/animoto.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/animoto\.com\/play\/\w+/i,
 
diff --git a/plugins/domains/bandcamp.com.js b/plugins/domains/bandcamp.com.js
index 12bc2108d..547908ff4 100644
--- a/plugins/domains/bandcamp.com.js
+++ b/plugins/domains/bandcamp.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/[a-z0-9-]+\.bandcamp\.com\/(album|track)\/(\w+)/i,
diff --git a/plugins/domains/beatport.com.js b/plugins/domains/beatport.com.js
index e77981099..f57971399 100644
--- a/plugins/domains/beatport.com.js
+++ b/plugins/domains/beatport.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www|pro)\.beatport\.com\/(track|mix)\/[a-zA-Z0-9\.\-]+\/(\d+)/i
diff --git a/plugins/domains/bigthink.com.js b/plugins/domains/bigthink.com.js
index d599e035e..702771309 100644
--- a/plugins/domains/bigthink.com.js
+++ b/plugins/domains/bigthink.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     mixins: [
         "*"
diff --git a/plugins/domains/box.com.js b/plugins/domains/box.com.js
index 66c1889c0..e1bcbbbf0 100644
--- a/plugins/domains/box.com.js
+++ b/plugins/domains/box.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https:\/\/app\.box\.com\/(?:embed|embed_widget)?\/?s\/([a-zA-Z0-9]+)/,
diff --git a/plugins/domains/brainyquote.com.js b/plugins/domains/brainyquote.com.js
index 2e93c9a94..09491f91c 100644
--- a/plugins/domains/brainyquote.com.js
+++ b/plugins/domains/brainyquote.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)?brainyquote\.com\/quotes(?:\/quotes)?\/\w\/?/i
diff --git a/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js b/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js
index 1e5367a38..150b36d37 100644
--- a/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js
+++ b/plugins/domains/buzzfeed.com/buzzfeed.isvideo.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.buzzfeed\.com\//i
diff --git a/plugins/domains/buzzfeed.com/buzzfeed.video.js b/plugins/domains/buzzfeed.com/buzzfeed.video.js
index 78369ab49..a4a3e65f0 100644
--- a/plugins/domains/buzzfeed.com/buzzfeed.video.js
+++ b/plugins/domains/buzzfeed.com/buzzfeed.video.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: '__promoUri',    
 
diff --git a/plugins/domains/c-span.org/c-span.org.js b/plugins/domains/c-span.org/c-span.org.js
index acd11cc54..147f98426 100644
--- a/plugins/domains/c-span.org/c-span.org.js
+++ b/plugins/domains/c-span.org/c-span.org.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.c-span\.org\/video\/\?(c?[\d-]+)(\/[\w-]+)/i,
 
diff --git a/plugins/domains/cbc.ca.js b/plugins/domains/cbc.ca.js
index 956bad3a6..5dc0a5b40 100644
--- a/plugins/domains/cbc.ca.js
+++ b/plugins/domains/cbc.ca.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.cbc\.ca\/player\//i,
diff --git a/plugins/domains/channel9.msdn.com.js b/plugins/domains/channel9.msdn.com.js
index b94b4b422..38b57e183 100644
--- a/plugins/domains/channel9.msdn.com.js
+++ b/plugins/domains/channel9.msdn.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/channel9\.msdn\.com\/Events\/([\w-]+\/\d+\/[\w-]+)/i,
 
diff --git a/plugins/domains/cnevids.com.js b/plugins/domains/cnevids.com.js
index d0aa03ab0..cc7773bfe 100644
--- a/plugins/domains/cnevids.com.js
+++ b/plugins/domains/cnevids.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     mixins: [
         "oembed-thumbnail",
diff --git a/plugins/domains/cnn.com.js b/plugins/domains/cnn.com.js
index 40f2e1f04..7345613d9 100644
--- a/plugins/domains/cnn.com.js
+++ b/plugins/domains/cnn.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(www|edition)?\.?cnn\.com\/videos?\//i,
diff --git a/plugins/domains/d.pr.js b/plugins/domains/d.pr.js
index 3e8542754..852b43da8 100644
--- a/plugins/domains/d.pr.js
+++ b/plugins/domains/d.pr.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(\w+\.)?d\.pr(?:\/free)?\/(?:i|v|)\//i
diff --git a/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js b/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js
index 36d07521e..cf9203b6d 100644
--- a/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js
+++ b/plugins/domains/dailymail.co.uk/dailymail.embeddedvideo.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     // direct "share" links to players from DailyMail articles. They end with #v-1467332342001
     re: [        
diff --git a/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js b/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js
index a8cc52a61..8c5fdfb66 100644
--- a/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js
+++ b/plugins/domains/dailymail.co.uk/dailymail.galleryvideo.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [        
         /^https?:\/\/www\.dailymail\.co\.uk\/video\/\w+\/video\-(\d+)\//i
diff --git a/plugins/domains/dailymail.co.uk/mol.im.js b/plugins/domains/dailymail.co.uk/mol.im.js
index fed5cf0e5..1a51c2678 100644
--- a/plugins/domains/dailymail.co.uk/mol.im.js
+++ b/plugins/domains/dailymail.co.uk/mol.im.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     // #v-1467332342001 isn't passed by Dailymail URL shortener
     re: /^https:\/\/mol\.im\/a\/\d+(#v\-\d+)/i,
diff --git a/plugins/domains/dailymotion.com/dailymotion.playlist.js b/plugins/domains/dailymotion.com/dailymotion.playlist.js
index 6f529bc36..b80975b16 100644
--- a/plugins/domains/dailymotion.com/dailymotion.playlist.js
+++ b/plugins/domains/dailymotion.com/dailymotion.playlist.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     /**
      * Endpoint `http://www.dailymotion.com/services/oembed`
      * does not provide oembed results for playlist url currently
diff --git a/plugins/domains/dailymotion.com/dailymotion.swf.js b/plugins/domains/dailymotion.com/dailymotion.swf.js
index 8335ebedf..55b7a869b 100644
--- a/plugins/domains/dailymotion.com/dailymotion.swf.js
+++ b/plugins/domains/dailymotion.com/dailymotion.swf.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.dailymotion\.com\/(swf|embed)\/video\//i,
 
diff --git a/plugins/domains/documentcloud.org.js b/plugins/domains/documentcloud.org.js
index eb5610448..c03a3b1ad 100644
--- a/plugins/domains/documentcloud.org.js
+++ b/plugins/domains/documentcloud.org.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:www)?\.?documentcloud\.org\/documents?\/\d+/i,
 
diff --git a/plugins/domains/dribbble.com.js b/plugins/domains/dribbble.com.js
index a58f818c6..1f5bdd3a3 100644
--- a/plugins/domains/dribbble.com.js
+++ b/plugins/domains/dribbble.com.js
@@ -1,6 +1,6 @@
 const PROFILE_RE = /^https?:\/\/dribbble\.com\/([a-zA-Z0-9\-]+)(?:\?[^\/]+)?$/i;
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/dribbble\.com\/shots\/([a-zA-Z0-9\-]+)/i,
diff --git a/plugins/domains/ebaumsworld.com.js b/plugins/domains/ebaumsworld.com.js
index 2d5724e59..d998c7b45 100644
--- a/plugins/domains/ebaumsworld.com.js
+++ b/plugins/domains/ebaumsworld.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.ebaumsworld\.com\/video\/watch\/(\d+)/i,
diff --git a/plugins/domains/espn.com.js b/plugins/domains/espn.com.js
index ffb00ceae..bf18f76ae 100644
--- a/plugins/domains/espn.com.js
+++ b/plugins/domains/espn.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)?espn\.com?(?:\.\w{2})?\/video\/clip\?id=espn:(\d+)/i,
diff --git a/plugins/domains/facebook.com/facebook.page.js b/plugins/domains/facebook.com/facebook.page.js
index a9f35bf6d..fae39efe6 100644
--- a/plugins/domains/facebook.com/facebook.page.js
+++ b/plugins/domains/facebook.com/facebook.page.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(www|m)\.facebook\.com\/([^\/\?]+(?<!\.php))\/?(?:about|photos|videos|events|timeline|photos_stream)?\/?(?:\?[^\/\?]+)?$/i,
diff --git a/plugins/domains/facebook.com/facebook.post.js b/plugins/domains/facebook.com/facebook.post.js
index 67201d05d..154ab8dff 100644
--- a/plugins/domains/facebook.com/facebook.post.js
+++ b/plugins/domains/facebook.com/facebook.post.js
@@ -1,6 +1,6 @@
 const DEFAULT_WIDTH = 640;
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www|m|business)\.facebook\.com\/(permalink|story)\.php\?[^\/]+(\d{10,})/i,
diff --git a/plugins/domains/facebook.com/facebook.video.js b/plugins/domains/facebook.com/facebook.video.js
index 4502380be..6268592c2 100644
--- a/plugins/domains/facebook.com/facebook.video.js
+++ b/plugins/domains/facebook.com/facebook.video.js
@@ -1,6 +1,6 @@
 const DEFAULT_WIDTH = 640;
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www|business)\.facebook\.com\/video\/video\.php.*[\?&]v=(\d{5,})(?:$|&)/i,
diff --git a/plugins/domains/flickr.com/flickr.gallery.js b/plugins/domains/flickr.com/flickr.gallery.js
index 81b372db6..255924c96 100644
--- a/plugins/domains/flickr.com/flickr.gallery.js
+++ b/plugins/domains/flickr.com/flickr.gallery.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.flickr\.com(\/photos\/[@a-zA-Z0-9_\.\-]+\/(?:sets|albums)\/(\d+))/i,    
 
diff --git a/plugins/domains/flickr.com/flickr.photo.js b/plugins/domains/flickr.com/flickr.photo.js
index 20f176bc5..5df53766d 100644
--- a/plugins/domains/flickr.com/flickr.photo.js
+++ b/plugins/domains/flickr.com/flickr.photo.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.flickr\.com\/photos\/([@a-zA-Z0-9_\.\-]+)\/(\d+).*?$/i,
 
diff --git a/plugins/domains/geogebra.iframe.js b/plugins/domains/geogebra.iframe.js
index 9b090fe55..c707a2fb7 100644
--- a/plugins/domains/geogebra.iframe.js
+++ b/plugins/domains/geogebra.iframe.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:tube|www)\.geogebra\.org\/material\/\w+\/id\/([a-zA-Z0-9]+)/i,
 
diff --git a/plugins/domains/geogebra.org.js b/plugins/domains/geogebra.org.js
index ccaefb8e1..7d37a5f3b 100644
--- a/plugins/domains/geogebra.org.js
+++ b/plugins/domains/geogebra.org.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:tube|www)\.geogebra\.org\/(?:m|classic|graphing)\/(?:[a-zA-Z0-9]+)#material\/([a-zA-Z0-9#]+)/i,
diff --git a/plugins/domains/giphy.com.js b/plugins/domains/giphy.com.js
index 565428940..58739e6c9 100644
--- a/plugins/domains/giphy.com.js
+++ b/plugins/domains/giphy.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/giphy\.com\/(?:gifs|stickers)\/(?:[a-zA-Z0-9_-]+\-)?([a-z0-9\-]+)/i,
diff --git a/plugins/domains/github.gist.js b/plugins/domains/github.gist.js
index 1d851ca50..4254a8856 100644
--- a/plugins/domains/github.gist.js
+++ b/plugins/domains/github.gist.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/gist\.github\.com\/([\w\.\-]+\/)(\w+)(#([\w\.\-]+))?/i,
 
diff --git a/plugins/domains/google.com/accounts.google.com.js b/plugins/domains/google.com/accounts.google.com.js
index 19809c071..fed8d9bc8 100644
--- a/plugins/domains/google.com/accounts.google.com.js
+++ b/plugins/domains/google.com/accounts.google.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /https?:\/\/accounts\.google\.com\/ServiceLogin/i,
diff --git a/plugins/domains/google.com/google.calendar.js b/plugins/domains/google.com/google.calendar.js
index 8e94ba830..4f977b7e3 100644
--- a/plugins/domains/google.com/google.calendar.js
+++ b/plugins/domains/google.com/google.calendar.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /https?:\/\/(?:www|calendar)\.google\.com\/calendar\/embed\/?\?([^\/]+)/i
diff --git a/plugins/domains/google.com/google.custommap.js b/plugins/domains/google.com/google.custommap.js
index fefaac7fa..3b4da299b 100644
--- a/plugins/domains/google.com/google.custommap.js
+++ b/plugins/domains/google.com/google.custommap.js
@@ -1,6 +1,6 @@
 // Custom Google Maps
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)?google\.(?:com?\.)?[a-z]+\/maps\/d\/(?:edit|embed|viewer)\?(?:[^&]+&)*(mid)=([a-zA-Z0-9\.\-_]+)/i,
diff --git a/plugins/domains/google.com/google.maps.js b/plugins/domains/google.com/google.maps.js
index 1e6d8cbd0..dfbfd4a30 100644
--- a/plugins/domains/google.com/google.maps.js
+++ b/plugins/domains/google.com/google.maps.js
@@ -1,7 +1,7 @@
 // Covers new google maps. Not classic ones. Classic ones are handled by maps.google.com.js plugin
 // Docs are at https://developers.google.com/maps/documentation/embed/guide
 
-module.exports = {
+export default {
 
     re: [
         // place 
diff --git a/plugins/domains/google.com/google.streetview.js b/plugins/domains/google.com/google.streetview.js
index 793a6019c..c0b3af4bb 100644
--- a/plugins/domains/google.com/google.streetview.js
+++ b/plugins/domains/google.com/google.streetview.js
@@ -1,7 +1,7 @@
 // Covers Google Maps street view images only: 
 // https://developers.google.com/maps/documentation/streetview/intro
 
-module.exports = {
+export default {
 
     re: [
         // place 
diff --git a/plugins/domains/hockeydb.com.js b/plugins/domains/hockeydb.com.js
index 625c21101..789a1332f 100644
--- a/plugins/domains/hockeydb.com.js
+++ b/plugins/domains/hockeydb.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.hockeydb\.com\/ihdb\/stats\/pdisplay\.php\?pid=(\d+)/i,
 
diff --git a/plugins/domains/i.gfycat.com.js b/plugins/domains/i.gfycat.com.js
index 24fa38f4d..2807de952 100644
--- a/plugins/domains/i.gfycat.com.js
+++ b/plugins/domains/i.gfycat.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /https?:\/\/(?:giant|thumbs|zippy)\.gfycat\.com\/([a-zA-Z0-9]+)(?:\-mobile|\-size_restricted)?\.(?:webm|mp4|gif)$/i,
diff --git a/plugins/domains/i.gifs.com.js b/plugins/domains/i.gifs.com.js
index 03271791e..aa9fb684e 100644
--- a/plugins/domains/i.gifs.com.js
+++ b/plugins/domains/i.gifs.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /https?:\/\/j\.gifs\.com\/(\w+)\.gif$/i
diff --git a/plugins/domains/i.giphy.com.js b/plugins/domains/i.giphy.com.js
index 103c0bea9..c4d01fa1a 100644
--- a/plugins/domains/i.giphy.com.js
+++ b/plugins/domains/i.giphy.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /https?:\/\/i\.giphy\.com\/(\w+)\.gif(\?.*)?$/i,
diff --git a/plugins/domains/ignore.js b/plugins/domains/ignore.js
index 0a0bbb944..3eb30c8fb 100644
--- a/plugins/domains/ignore.js
+++ b/plugins/domains/ignore.js
@@ -2,7 +2,7 @@
 
 const RE = CONFIG.IGNORE_DOMAINS_RE || CONFIG.BLACKLIST_DOMAINS_RE;
 
-module.exports = {
+export default {
 
     re: RE,
 
diff --git a/plugins/domains/imageshack.com.js b/plugins/domains/imageshack.com.js
index ada5358dd..7e7e957d5 100644
--- a/plugins/domains/imageshack.com.js
+++ b/plugins/domains/imageshack.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     re: /^https?:\/\/imageshack\.com\/i\/\w+/i,
 
     mixins: [
diff --git a/plugins/domains/imdb.com.js b/plugins/domains/imdb.com.js
index ac1f7990c..d067c69b6 100644
--- a/plugins/domains/imdb.com.js
+++ b/plugins/domains/imdb.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.imdb\.com\/video\/(?:[\w]+\/)?vi(\d+)/i,
diff --git a/plugins/domains/issuu.com.js b/plugins/domains/issuu.com.js
index 33c43532f..7afa45739 100644
--- a/plugins/domains/issuu.com.js
+++ b/plugins/domains/issuu.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/issuu\.com\/[\w_.-]+\/docs\/([\w_.-]+)/i,
 
diff --git a/plugins/domains/jsfiddle.net.js b/plugins/domains/jsfiddle.net.js
index 6dfbe6e2d..85048acda 100644
--- a/plugins/domains/jsfiddle.net.js
+++ b/plugins/domains/jsfiddle.net.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^(https?:\/\/jsfiddle.net\/(?:\w+\/)?\w+\/)\/?(?:[^\/]+)?$/i,
 
diff --git a/plugins/domains/knightlab.js b/plugins/domains/knightlab.js
index 7d16c9289..2e152a53e 100644
--- a/plugins/domains/knightlab.js
+++ b/plugins/domains/knightlab.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/s3\.amazonaws\.com\/(?:uploads|CDN)\.knightlab\.com\/(storymapjs)\/\w+\/(?:[a-zA-Z0-9\-\/]+)\.html/i,
diff --git a/plugins/domains/libsyn.com.js b/plugins/domains/libsyn.com.js
index 67c46e27c..8883e157c 100644
--- a/plugins/domains/libsyn.com.js
+++ b/plugins/domains/libsyn.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/[a-zA-Z0-9\.]+\.libsyn(?:pro)?\.com\//,
 
diff --git a/plugins/domains/live.amcharts.com.js b/plugins/domains/live.amcharts.com.js
index 8f08b7890..0f8828f69 100644
--- a/plugins/domains/live.amcharts.com.js
+++ b/plugins/domains/live.amcharts.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/live\.amcharts\.com\/([a-zA-Z0-9\-]+)/i,
 
diff --git a/plugins/domains/livestream.com.js b/plugins/domains/livestream.com.js
index 2fdb6ed67..80f6efa56 100644
--- a/plugins/domains/livestream.com.js
+++ b/plugins/domains/livestream.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/livestream\.com\/(\w+)\/events\/(\d+)(\/videos?\/\d+)?/i,
diff --git a/plugins/domains/mail.ru.js b/plugins/domains/mail.ru.js
index eb665bc0e..89cfe0993 100644
--- a/plugins/domains/mail.ru.js
+++ b/plugins/domains/mail.ru.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/my\.mail\.ru\/\/?[a-z\.]+\/[a-zA-Z0-9\._\-]+\/video\/([a-zA-Z0-9_]+)\/([a-zA-Z0-9_]+)\.html/i
diff --git a/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js b/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js
index 9641ff518..5225db92d 100644
--- a/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js
+++ b/plugins/domains/maps.yandex.ru/maps.yandex.ru.short.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https:\/\/yandex\.ru\/maps\/-\//,
 
diff --git a/plugins/domains/medium.com.js b/plugins/domains/medium.com.js
index 663f58d76..bf1ba5d1e 100644
--- a/plugins/domains/medium.com.js
+++ b/plugins/domains/medium.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https:\/\/(?:[a-z0-9\-]+\.)?medium\.com\/@?[\w-]+/i,
 
diff --git a/plugins/domains/momindum.com.js b/plugins/domains/momindum.com.js
index 13b441407..2dbc2e334 100644
--- a/plugins/domains/momindum.com.js
+++ b/plugins/domains/momindum.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     mixins: [
         "*"
diff --git a/plugins/domains/nbcnews.com.js b/plugins/domains/nbcnews.com.js
index 74e1db79a..658b7cfa4 100644
--- a/plugins/domains/nbcnews.com.js
+++ b/plugins/domains/nbcnews.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.nbcnews\.com\/(?:[a-z\-]+\/)?videos?\/[a-zA-Z0-9-]+\-(\d+)/i,
 
diff --git a/plugins/domains/nbcsports.com.js b/plugins/domains/nbcsports.com.js
index 5387d4e0d..000020ba5 100644
--- a/plugins/domains/nbcsports.com.js
+++ b/plugins/domains/nbcsports.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.nbcsports\.com\/videos?\/[a-zA-Z0-9-]+/i
diff --git a/plugins/domains/nhl.com/nhl.com.js b/plugins/domains/nhl.com/nhl.com.js
index 2e79997e8..458569ad6 100644
--- a/plugins/domains/nhl.com/nhl.com.js
+++ b/plugins/domains/nhl.com/nhl.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /https?:\/\/(?:www\.)?nhl\.com(\/\w+\/)video\/(?:embed\/)?([a-zA-Z0-9\-]+\/t\-\d+\/c\-\d+)/i,
diff --git a/plugins/domains/npr.org/npr.sections.js b/plugins/domains/npr.org/npr.sections.js
index e79882272..b2fa55989 100644
--- a/plugins/domains/npr.org/npr.sections.js
+++ b/plugins/domains/npr.org/npr.sections.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.npr\.org\/sections\//i,
diff --git a/plugins/domains/nytimes.video.js b/plugins/domains/nytimes.video.js
index 01a875c51..bda5ec89d 100644
--- a/plugins/domains/nytimes.video.js
+++ b/plugins/domains/nytimes.video.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.nytimes\.com\/video\/[a-zA-Z0-9\-]+\/(\d+)\//,
diff --git a/plugins/domains/ow.ly.js b/plugins/domains/ow.ly.js
index 27e6b6dce..da63cbd82 100644
--- a/plugins/domains/ow.ly.js
+++ b/plugins/domains/ow.ly.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/ow\.ly\/i\//i
diff --git a/plugins/domains/padlet.com.js b/plugins/domains/padlet.com.js
index 910f80db3..88adbb2f8 100644
--- a/plugins/domains/padlet.com.js
+++ b/plugins/domains/padlet.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/padlet\.com\/[0-9a-zA-Z_\-]+\/([0-9a-zA-Z_\-]+)/i
diff --git a/plugins/domains/pastebin.com.js b/plugins/domains/pastebin.com.js
index d55669639..d08c8c12c 100644
--- a/plugins/domains/pastebin.com.js
+++ b/plugins/domains/pastebin.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/pastebin\.com\/(?!search)([a-zA-Z0-9]+)/i
diff --git a/plugins/domains/piktochart.com/piktochart.com.js b/plugins/domains/piktochart.com/piktochart.com.js
index e6efc9f66..2e954a64d 100644
--- a/plugins/domains/piktochart.com/piktochart.com.js
+++ b/plugins/domains/piktochart.com/piktochart.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:magic|create)\.piktochart\.com\/output\/(\d+\-[\-a-zA-Z-0-9_]+)/i,
 
diff --git a/plugins/domains/pinterest.com/pinterest.board.js b/plugins/domains/pinterest.com/pinterest.board.js
index 158d7d500..4f4bd8f5e 100644
--- a/plugins/domains/pinterest.com/pinterest.board.js
+++ b/plugins/domains/pinterest.com/pinterest.board.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:\w{2,3}\.)?pinterest(?:\.com?)?\.\w{2,3}\/((?!pin)[a-zA-Z0-9%_]+|pinterest)\/([a-zA-Z0-9%\-]+)\/?(?:$|\?|#)/i,
 
diff --git a/plugins/domains/pinterest.com/pinterest.pin.js b/plugins/domains/pinterest.com/pinterest.pin.js
index 56def9f4d..564c73cbb 100644
--- a/plugins/domains/pinterest.com/pinterest.pin.js
+++ b/plugins/domains/pinterest.com/pinterest.pin.js
@@ -1,5 +1,5 @@
 
-module.exports = {
+export default {
 
     re: /^(https?:\/\/(?:\w{2,3}\.)?pinterest(?:\.com?)?\.\w{2,3})\/pin\/(?:[^\/]+\-)?(\d+)/i,
 
diff --git a/plugins/domains/pinterest.com/pinterest.user.js b/plugins/domains/pinterest.com/pinterest.user.js
index bafa21366..266546af1 100644
--- a/plugins/domains/pinterest.com/pinterest.user.js
+++ b/plugins/domains/pinterest.com/pinterest.user.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:\w{2,3}\.)?pinterest(?:\.com?)?\.\w{2,3}\/((?!pin)[a-zA-Z0-9%_]+|pinterest)\/?(?:$|\?|#)/i,
 
diff --git a/plugins/domains/polldaddy.com.js b/plugins/domains/polldaddy.com.js
index b737898f8..f534adb10 100644
--- a/plugins/domains/polldaddy.com.js
+++ b/plugins/domains/polldaddy.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:\w+\.)?polldaddy\.com\/poll\/([0-9]+)/i,
diff --git a/plugins/domains/prezi.com.js b/plugins/domains/prezi.com.js
index cb460d533..dc1f6fd3c 100644
--- a/plugins/domains/prezi.com.js
+++ b/plugins/domains/prezi.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https:\/\/prezi\.com\/([a-z0-9\-_]+)\/[^\/]+\/(?:\?[^\/\?]+)?$/i,
 
diff --git a/plugins/domains/quizlet.com.js b/plugins/domains/quizlet.com.js
index b29a759fb..d4dc4cf03 100644
--- a/plugins/domains/quizlet.com.js
+++ b/plugins/domains/quizlet.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/quizlet\.com\/(?:\w{2}\/)?(\d+)\/([^\/]+)\/?/i
diff --git a/plugins/domains/rapgenius.com.js b/plugins/domains/rapgenius.com.js
index 7b469de82..f90770575 100644
--- a/plugins/domains/rapgenius.com.js
+++ b/plugins/domains/rapgenius.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:[\w\-]+\.)?genius\.com\/(?!jobs)([a-zA-Z0-9\-]+)/i,
 
diff --git a/plugins/domains/readymag.com.js b/plugins/domains/readymag.com.js
index 773912e5d..563c9b702 100644
--- a/plugins/domains/readymag.com.js
+++ b/plugins/domains/readymag.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/readymag\.com\/\w+\/(\d+)/i
diff --git a/plugins/domains/simplecast.com.js b/plugins/domains/simplecast.com.js
index a2c5ddcb8..dce05bbf9 100644
--- a/plugins/domains/simplecast.com.js
+++ b/plugins/domains/simplecast.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     // also used for hosted simplecasts
     re: [
diff --git a/plugins/domains/slid.es.js b/plugins/domains/slid.es.js
index 2849074c9..37c731a74 100644
--- a/plugins/domains/slid.es.js
+++ b/plugins/domains/slid.es.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     re: [
         /^https?:\/\/slides\.com\/([a-zA-Z0-9_\-]+)\/([a-zA-Z0-9_\-]+)/i
     ],
diff --git a/plugins/domains/smugmug.com.js b/plugins/domains/smugmug.com.js
index f67ac9f47..3d4f2cdc2 100644
--- a/plugins/domains/smugmug.com.js
+++ b/plugins/domains/smugmug.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     mixins: [
         "*"
diff --git a/plugins/domains/soundcloud.com/soundcloud-fallbacks.js b/plugins/domains/soundcloud.com/soundcloud-fallbacks.js
index 1c1f69a2d..46dde6e9a 100644
--- a/plugins/domains/soundcloud.com/soundcloud-fallbacks.js
+++ b/plugins/domains/soundcloud.com/soundcloud-fallbacks.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: ['sound'],
 
diff --git a/plugins/domains/soundcloud.com/soundcloud-oembed-error.js b/plugins/domains/soundcloud.com/soundcloud-oembed-error.js
index 38a2cc529..c4beb1ce9 100644
--- a/plugins/domains/soundcloud.com/soundcloud-oembed-error.js
+++ b/plugins/domains/soundcloud.com/soundcloud-oembed-error.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: ['__allow_soundcloud_meta', 'iframe'],
 
diff --git a/plugins/domains/spotify.com.js b/plugins/domains/spotify.com.js
index f11d9fca6..fa5856b10 100644
--- a/plugins/domains/spotify.com.js
+++ b/plugins/domains/spotify.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:open|play|www)\.spotify\.com\/(?:track|album|artist|show|episode|playlist)/i
diff --git a/plugins/domains/sverigesradio.se.js b/plugins/domains/sverigesradio.se.js
index 4fb71b863..b162b678a 100644
--- a/plugins/domains/sverigesradio.se.js
+++ b/plugins/domains/sverigesradio.se.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     // This is for articles only - players are covered by oembed auto-discovery
     re: [
diff --git a/plugins/domains/ted.com/ted.playlists.js b/plugins/domains/ted.com/ted.playlists.js
index f28c0a003..a7cc8b064 100644
--- a/plugins/domains/ted.com/ted.playlists.js
+++ b/plugins/domains/ted.com/ted.playlists.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:www\.)?ted\.com\/playlists\//i,
 
diff --git a/plugins/domains/test.com/test1.js b/plugins/domains/test.com/test1.js
index d82cda261..01ef363d1 100644
--- a/plugins/domains/test.com/test1.js
+++ b/plugins/domains/test.com/test1.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: 'test_data1',
 
diff --git a/plugins/domains/test.com/test2.js b/plugins/domains/test.com/test2.js
index 519bfb8cf..33fd1557c 100644
--- a/plugins/domains/test.com/test2.js
+++ b/plugins/domains/test.com/test2.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: ['test_data2', 'test_plugin'],
 
diff --git a/plugins/domains/test.com/test3.js b/plugins/domains/test.com/test3.js
index 45843d327..7e0d746cc 100644
--- a/plugins/domains/test.com/test3.js
+++ b/plugins/domains/test.com/test3.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(test_data2) {
         return {
diff --git a/plugins/domains/theguardian.com/guardian.video.js b/plugins/domains/theguardian.com/guardian.video.js
index 9e2ebcafd..352d42157 100644
--- a/plugins/domains/theguardian.com/guardian.video.js
+++ b/plugins/domains/theguardian.com/guardian.video.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /https?:\/\/www\.theguardian\.com\/[\w-]+\/video\/\d+\/\w+\/\d+\/[\w-]+/i,
 
diff --git a/plugins/domains/theta360.com/theta360.com.js b/plugins/domains/theta360.com/theta360.com.js
index 7f6c185b4..0cdffe9c3 100644
--- a/plugins/domains/theta360.com/theta360.com.js
+++ b/plugins/domains/theta360.com/theta360.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /https:\/\/theta360\.com\/(?:spheres|s)(?:\/\w+)?\/[\w-]+/i,
 
diff --git a/plugins/domains/tmz.com.js b/plugins/domains/tmz.com.js
index 345a1040d..fcca586c1 100644
--- a/plugins/domains/tmz.com.js
+++ b/plugins/domains/tmz.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.tmz\.com\/videos\/([a-zA-Z0-9_]+)/,
 
diff --git a/plugins/domains/twitch.tv/clips.twitch.tv.js b/plugins/domains/twitch.tv/clips.twitch.tv.js
index e76ced53b..21ab7d97c 100644
--- a/plugins/domains/twitch.tv/clips.twitch.tv.js
+++ b/plugins/domains/twitch.tv/clips.twitch.tv.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/clips.twitch\.tv\/([^\?\/]+)(?:\?[^\?]+)?$/i,
diff --git a/plugins/domains/twitch.tv/twitch-live-fallback.js b/plugins/domains/twitch.tv/twitch-live-fallback.js
index 84431a733..557c3005f 100644
--- a/plugins/domains/twitch.tv/twitch-live-fallback.js
+++ b/plugins/domains/twitch.tv/twitch-live-fallback.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.|go\.)?twitch\.tv\/([a-zA-Z0-9_]+)(?:\?parent=.*)?$/i,
diff --git a/plugins/domains/twitter.com/twitter.timelines.js b/plugins/domains/twitter.com/twitter.timelines.js
index 2ed3fb7fd..561e8f41e 100644
--- a/plugins/domains/twitter.com/twitter.timelines.js
+++ b/plugins/domains/twitter.com/twitter.timelines.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     // Embedded Like, Collection, and Moment Timelines are now retired.
     // https://twittercommunity.com/t/removing-support-for-embedded-like-collection-and-moment-timelines/150313
diff --git a/plugins/domains/usatoday.com.js b/plugins/domains/usatoday.com.js
index d6caf9a6e..605f52ba5 100644
--- a/plugins/domains/usatoday.com.js
+++ b/plugins/domains/usatoday.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.([a-z\-]+)\.com\/media\/cinematic\/video\/(\d{7,12})\/[a-zA-Z0-9\-\_:\.]+\/?(?:[^\/]+)?$/i,
diff --git a/plugins/domains/vbox7.com.js b/plugins/domains/vbox7.com.js
index 007b67346..3e1ed0334 100644
--- a/plugins/domains/vbox7.com.js
+++ b/plugins/domains/vbox7.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [        
         /https?:\/\/(?:www\.)?vbox7\.com\/play:(\w+)/i
diff --git a/plugins/domains/vimeo-moogaloop.js b/plugins/domains/vimeo-moogaloop.js
index 2d43c1f46..edcb01730 100644
--- a/plugins/domains/vimeo-moogaloop.js
+++ b/plugins/domains/vimeo-moogaloop.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/vimeo\.com\/moogaloop\.swf\?clip_id=(\d+)/i,
 
diff --git a/plugins/domains/washingtonpost.com.js b/plugins/domains/washingtonpost.com.js
index 2ebe6450d..1042f07fc 100644
--- a/plugins/domains/washingtonpost.com.js
+++ b/plugins/domains/washingtonpost.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https:\/\/www\.washingtonpost\.com\/(?:video|posttv)\/\w+\/[a-z0-9-]+\/\d+\/\d+\/\d+\/([a-z0-9-]+)_video\.html/i,
 
diff --git a/plugins/domains/widgetic.com.js b/plugins/domains/widgetic.com.js
index 9af8e8c5f..78b1b8167 100644
--- a/plugins/domains/widgetic.com.js
+++ b/plugins/domains/widgetic.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: [
         /https?:\/\/widgetic\.com\/widgets\//i,
diff --git a/plugins/domains/wistia.com.js b/plugins/domains/wistia.com.js
index 3bf63667b..796489b46 100644
--- a/plugins/domains/wistia.com.js
+++ b/plugins/domains/wistia.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/([a-zA-Z0-9]+).wistia\.com\/medias\/([_a-zA-Z0-9]+)/i,
 
diff --git a/plugins/domains/youku.com.js b/plugins/domains/youku.com.js
index ff3c36a45..96dd4c3cb 100644
--- a/plugins/domains/youku.com.js
+++ b/plugins/domains/youku.com.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     //http://v.youku.com/v_show/id_XNDkwNjg2NzQw.html?f=18736842
     re: [
diff --git a/plugins/domains/youtube.com/youtube.shared.js b/plugins/domains/youtube.com/youtube.shared.js
index 86dd1d5a2..3b8778862 100644
--- a/plugins/domains/youtube.com/youtube.shared.js
+++ b/plugins/domains/youtube.com/youtube.shared.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     re: /^https?:\/\/www\.youtube\.com\/shared\/?\?ci=([a-zA-Z0-9_-]+)/i,
 
diff --git a/plugins/links/article/check-article.js b/plugins/links/article/check-article.js
index 10e77f3d8..11185238e 100644
--- a/plugins/links/article/check-article.js
+++ b/plugins/links/article/check-article.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: '__is_general_article',
 
diff --git a/plugins/links/canonical-redirect.js b/plugins/links/canonical-redirect.js
index 679260b11..36f8fb4c5 100644
--- a/plugins/links/canonical-redirect.js
+++ b/plugins/links/canonical-redirect.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(url, meta, options, cb) {
 
diff --git a/plugins/links/embedURL/checkEmbedURL.js b/plugins/links/embedURL/checkEmbedURL.js
index 350793ecc..beb89965e 100644
--- a/plugins/links/embedURL/checkEmbedURL.js
+++ b/plugins/links/embedURL/checkEmbedURL.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: '__allowEmbedURL',
 
diff --git a/plugins/links/favicon.js b/plugins/links/favicon.js
index 5817d3250..367d95aac 100644
--- a/plugins/links/favicon.js
+++ b/plugins/links/favicon.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(meta) {
 
diff --git a/plugins/links/hosted/brightcove-allow-in-page.js b/plugins/links/hosted/brightcove-allow-in-page.js
index 43a4b0b18..97c1b13a1 100644
--- a/plugins/links/hosted/brightcove-allow-in-page.js
+++ b/plugins/links/hosted/brightcove-allow-in-page.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: '__allowBrightcoveInPage',
 
diff --git a/plugins/links/hosted/brightcove-hosted.js b/plugins/links/hosted/brightcove-hosted.js
index c62629355..11cca8688 100644
--- a/plugins/links/hosted/brightcove-hosted.js
+++ b/plugins/links/hosted/brightcove-hosted.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: '__promoUri',
 
diff --git a/plugins/links/hosted/brightcove-in-page-promo.js b/plugins/links/hosted/brightcove-in-page-promo.js
index 2165ff3f2..d159ea295 100644
--- a/plugins/links/hosted/brightcove-in-page-promo.js
+++ b/plugins/links/hosted/brightcove-in-page-promo.js
@@ -3,7 +3,7 @@
  * http://docs.brightcove.com/en/perform/brightcove-player/guides/embed-in-page.html
  */
 
-module.exports = {
+export default {
 
     provides: '__promoUri',
 
diff --git a/plugins/links/hosted/cloudapp-hosted.js b/plugins/links/hosted/cloudapp-hosted.js
index f79d77485..8e16540ca 100644
--- a/plugins/links/hosted/cloudapp-hosted.js
+++ b/plugins/links/hosted/cloudapp-hosted.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(twitter) {
 
diff --git a/plugins/links/hosted/cnevids-hosted.js b/plugins/links/hosted/cnevids-hosted.js
index 86ae5f9e5..91fda2f30 100644
--- a/plugins/links/hosted/cnevids-hosted.js
+++ b/plugins/links/hosted/cnevids-hosted.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getData: function(video_src) {
 
diff --git a/plugins/links/hosted/hosted-via-canonical.js b/plugins/links/hosted/hosted-via-canonical.js
index 343dfb785..4f3385190 100644
--- a/plugins/links/hosted/hosted-via-canonical.js
+++ b/plugins/links/hosted/hosted-via-canonical.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(url, meta, options, cb) {
 
diff --git a/plugins/links/hosted/known-players.js b/plugins/links/hosted/known-players.js
index b88991a7b..1b1a1fdb5 100644
--- a/plugins/links/hosted/known-players.js
+++ b/plugins/links/hosted/known-players.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: '__promoUri',
 
diff --git a/plugins/links/hosted/libsyn-hosted.js b/plugins/links/hosted/libsyn-hosted.js
index b65a4cfb4..cb86ac0fe 100644
--- a/plugins/links/hosted/libsyn-hosted.js
+++ b/plugins/links/hosted/libsyn-hosted.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getData: function(url, video_src, twitter) {
 
diff --git a/plugins/links/hosted/podbean-hosted.js b/plugins/links/hosted/podbean-hosted.js
index e17ec1b11..23ff977ce 100644
--- a/plugins/links/hosted/podbean-hosted.js
+++ b/plugins/links/hosted/podbean-hosted.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getData: function(video_src) {
 
diff --git a/plugins/links/hosted/promo-link.js b/plugins/links/hosted/promo-link.js
index e32041623..627cd62be 100644
--- a/plugins/links/hosted/promo-link.js
+++ b/plugins/links/hosted/promo-link.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: '__promoUri',
 
diff --git a/plugins/links/hosted/tout-hosted.js b/plugins/links/hosted/tout-hosted.js
index 3104815e3..3a1df1cb9 100644
--- a/plugins/links/hosted/tout-hosted.js
+++ b/plugins/links/hosted/tout-hosted.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(twitter, whitelistRecord) {
 
diff --git a/plugins/links/hosted/video_src-hosted.js b/plugins/links/hosted/video_src-hosted.js
index fb40678ea..d8feea74c 100644
--- a/plugins/links/hosted/video_src-hosted.js
+++ b/plugins/links/hosted/video_src-hosted.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: 'video_src',
 
diff --git a/plugins/links/logo.js b/plugins/links/logo.js
index b3d51a834..902cb7ca7 100644
--- a/plugins/links/logo.js
+++ b/plugins/links/logo.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(meta) {
 
diff --git a/plugins/links/ms-doc-viewer.js b/plugins/links/ms-doc-viewer.js
index f664db77e..77cffa9af 100644
--- a/plugins/links/ms-doc-viewer.js
+++ b/plugins/links/ms-doc-viewer.js
@@ -1,6 +1,6 @@
 // https://support.office.com/en-gb/article/View-Office-documents-online-1cc2ea26-0f7b-41f7-8e0e-6461a104544e?ui=en-US&rs=en-GB&ad=GB
 
-module.exports = {
+export default {
 
     getLink: function(url, __nonHtmlContentData, options) {
 
diff --git a/plugins/links/oembed-icon.js b/plugins/links/oembed-icon.js
index 5be21e978..34e6e7033 100644
--- a/plugins/links/oembed-icon.js
+++ b/plugins/links/oembed-icon.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(oembed) {
 
diff --git a/plugins/links/oembed-photo.js b/plugins/links/oembed-photo.js
index 53734bffb..d51af72e7 100644
--- a/plugins/links/oembed-photo.js
+++ b/plugins/links/oembed-photo.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(oembed, whitelistRecord, url) {
 
diff --git a/plugins/links/oembed-rich.js b/plugins/links/oembed-rich.js
index 3a214f9bd..b6817f16c 100644
--- a/plugins/links/oembed-rich.js
+++ b/plugins/links/oembed-rich.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(oembed, whitelistRecord, url) {
 
diff --git a/plugins/links/oembed-thumbnail.js b/plugins/links/oembed-thumbnail.js
index d9dee6097..5bb677fb3 100644
--- a/plugins/links/oembed-thumbnail.js
+++ b/plugins/links/oembed-thumbnail.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(oembed) {
         if (oembed.thumbnail_url) {
diff --git a/plugins/links/parsely-image.js b/plugins/links/parsely-image.js
index 52ef44453..67f6976bc 100644
--- a/plugins/links/parsely-image.js
+++ b/plugins/links/parsely-image.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(parsely) {
         return {
diff --git a/plugins/links/prerender/utils.js b/plugins/links/prerender/utils.js
index 9f1717238..b53e0a4da 100644
--- a/plugins/links/prerender/utils.js
+++ b/plugins/links/prerender/utils.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     notPlugin: true,
 
diff --git a/plugins/links/twitter-image.js b/plugins/links/twitter-image.js
index 023a3c665..01dc7fd2b 100644
--- a/plugins/links/twitter-image.js
+++ b/plugins/links/twitter-image.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLinks: function(twitter, meta, whitelistRecord) {
 
diff --git a/plugins/links/twitter-player.js b/plugins/links/twitter-player.js
index 6300ee611..e4376936e 100644
--- a/plugins/links/twitter-player.js
+++ b/plugins/links/twitter-player.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function(twitter, whitelistRecord) {
 
diff --git a/plugins/links/video_src.js b/plugins/links/video_src.js
index 2c9fb4e63..9a135f55c 100644
--- a/plugins/links/video_src.js
+++ b/plugins/links/video_src.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getLink: function (meta, whitelistRecord) {
 
diff --git a/plugins/meta/amphtml.js b/plugins/meta/amphtml.js
index 33aa89365..c70244f69 100644
--- a/plugins/meta/amphtml.js
+++ b/plugins/meta/amphtml.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
         return {
diff --git a/plugins/meta/author.js b/plugins/meta/author.js
index e903f266e..3b80995ce 100644
--- a/plugins/meta/author.js
+++ b/plugins/meta/author.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/canonical.js b/plugins/meta/canonical.js
index 619e55616..0699849f1 100644
--- a/plugins/meta/canonical.js
+++ b/plugins/meta/canonical.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     highestPriority: true,
 
diff --git a/plugins/meta/category.js b/plugins/meta/category.js
index 688f98988..dfc825a62 100644
--- a/plugins/meta/category.js
+++ b/plugins/meta/category.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
         return {
diff --git a/plugins/meta/copyright.js b/plugins/meta/copyright.js
index 6113772d5..b82a95318 100644
--- a/plugins/meta/copyright.js
+++ b/plugins/meta/copyright.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
         return {
diff --git a/plugins/meta/date.js b/plugins/meta/date.js
index 21a15e7d6..15cbaf061 100644
--- a/plugins/meta/date.js
+++ b/plugins/meta/date.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta, url) {
 
diff --git a/plugins/meta/dc-title.js b/plugins/meta/dc-title.js
index 954c8ec4a..3727357e8 100644
--- a/plugins/meta/dc-title.js
+++ b/plugins/meta/dc-title.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/dc.js b/plugins/meta/dc.js
index efeb73c5e..2ea88bc8a 100644
--- a/plugins/meta/dc.js
+++ b/plugins/meta/dc.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/description.js b/plugins/meta/description.js
index 8893a4f36..71c01d0b9 100644
--- a/plugins/meta/description.js
+++ b/plugins/meta/description.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     lowestPriority: true,
 
diff --git a/plugins/meta/embedurl-meta.js b/plugins/meta/embedurl-meta.js
index c730c88c6..40a6d78b3 100644
--- a/plugins/meta/embedurl-meta.js
+++ b/plugins/meta/embedurl-meta.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     lowestPriority: true,
 
diff --git a/plugins/meta/geo-url.js b/plugins/meta/geo-url.js
index a222927e4..3c5080a81 100644
--- a/plugins/meta/geo-url.js
+++ b/plugins/meta/geo-url.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/html-title.js b/plugins/meta/html-title.js
index f56dd7085..8f6ff3b13 100644
--- a/plugins/meta/html-title.js
+++ b/plugins/meta/html-title.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
         return {
diff --git a/plugins/meta/keywords.js b/plugins/meta/keywords.js
index eea548b85..85bed842f 100644
--- a/plugins/meta/keywords.js
+++ b/plugins/meta/keywords.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/ld-article-keywords.js b/plugins/meta/ld-article-keywords.js
index be11d70d5..d77b6c4a0 100644
--- a/plugins/meta/ld-article-keywords.js
+++ b/plugins/meta/ld-article-keywords.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(ld) {
 
diff --git a/plugins/meta/meta-music.js b/plugins/meta/meta-music.js
index 464e1114c..31efeba6e 100644
--- a/plugins/meta/meta-music.js
+++ b/plugins/meta/meta-music.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
         if (meta.music) {
diff --git a/plugins/meta/meta-title.js b/plugins/meta/meta-title.js
index 03b214430..f99d26ce3 100644
--- a/plugins/meta/meta-title.js
+++ b/plugins/meta/meta-title.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/noscript-redirect.js b/plugins/meta/noscript-redirect.js
index 4f2857271..b5093565b 100644
--- a/plugins/meta/noscript-redirect.js
+++ b/plugins/meta/noscript-redirect.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getData: function(url, meta, cb) {
         return cb(
diff --git a/plugins/meta/oembed-author.js b/plugins/meta/oembed-author.js
index 20be0f028..2acb39c92 100644
--- a/plugins/meta/oembed-author.js
+++ b/plugins/meta/oembed-author.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     highestPriority: true,
 
diff --git a/plugins/meta/oembed-canonical.js b/plugins/meta/oembed-canonical.js
index 6c6aacb85..f4bb390cc 100644
--- a/plugins/meta/oembed-canonical.js
+++ b/plugins/meta/oembed-canonical.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(oembed) {
         if (oembed.type != "photo") {
diff --git a/plugins/meta/oembed-duration.js b/plugins/meta/oembed-duration.js
index 7d6515f81..db94d3dd9 100644
--- a/plugins/meta/oembed-duration.js
+++ b/plugins/meta/oembed-duration.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(oembed) {
         return {
diff --git a/plugins/meta/oembed-license.js b/plugins/meta/oembed-license.js
index 70316e2d3..10cd9a327 100644
--- a/plugins/meta/oembed-license.js
+++ b/plugins/meta/oembed-license.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(oembed) {
         return {
diff --git a/plugins/meta/oembed-product.js b/plugins/meta/oembed-product.js
index bee73397d..29db456e3 100644
--- a/plugins/meta/oembed-product.js
+++ b/plugins/meta/oembed-product.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(oembed) {
 
diff --git a/plugins/meta/oembed-site.js b/plugins/meta/oembed-site.js
index 4d52bd80c..f4f9dcc29 100644
--- a/plugins/meta/oembed-site.js
+++ b/plugins/meta/oembed-site.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(oembed) {
         return {
diff --git a/plugins/meta/oembed-title.js b/plugins/meta/oembed-title.js
index b6edfc280..012a16d22 100644
--- a/plugins/meta/oembed-title.js
+++ b/plugins/meta/oembed-title.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     // avoid wordpress pages
     lowestPriority: true,
diff --git a/plugins/meta/og-article.js b/plugins/meta/og-article.js
index 2a8abbccc..b563e4f61 100644
--- a/plugins/meta/og-article.js
+++ b/plugins/meta/og-article.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/og-date.js b/plugins/meta/og-date.js
index 1c99ab726..6a6b46f69 100644
--- a/plugins/meta/og-date.js
+++ b/plugins/meta/og-date.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(og) {
 
diff --git a/plugins/meta/og-description.js b/plugins/meta/og-description.js
index 8012f415a..5654db4ce 100644
--- a/plugins/meta/og-description.js
+++ b/plugins/meta/og-description.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(og) {
 
diff --git a/plugins/meta/og-location.js b/plugins/meta/og-location.js
index b66b99e51..b832ee4ca 100644
--- a/plugins/meta/og-location.js
+++ b/plugins/meta/og-location.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(og) {
 
diff --git a/plugins/meta/og-product.js b/plugins/meta/og-product.js
index 8b84fd0e1..e0514af6a 100644
--- a/plugins/meta/og-product.js
+++ b/plugins/meta/og-product.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     // https://developers.pinterest.com/docs/rich-pins/products/?
     getMeta: function(og, meta) {
diff --git a/plugins/meta/og-site.js b/plugins/meta/og-site.js
index c2c7a34a7..8d648edce 100644
--- a/plugins/meta/og-site.js
+++ b/plugins/meta/og-site.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(og) {
 
diff --git a/plugins/meta/og-title.js b/plugins/meta/og-title.js
index e0e32ae56..fa5a67fc3 100644
--- a/plugins/meta/og-title.js
+++ b/plugins/meta/og-title.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(og) {
 
diff --git a/plugins/meta/og-video-duration.js b/plugins/meta/og-video-duration.js
index c67b40930..61eaf8a1d 100644
--- a/plugins/meta/og-video-duration.js
+++ b/plugins/meta/og-video-duration.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(og) {
 
diff --git a/plugins/meta/parsely.js b/plugins/meta/parsely.js
index e32682743..f9f82c8bc 100644
--- a/plugins/meta/parsely.js
+++ b/plugins/meta/parsely.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     highestPriority: true,
 
diff --git a/plugins/meta/shortlink.js b/plugins/meta/shortlink.js
index 3e041f7a8..269304106 100644
--- a/plugins/meta/shortlink.js
+++ b/plugins/meta/shortlink.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
     getMeta: function(meta) {
 
         var s = meta.shorturl || meta.short_url || meta.shortlink;
diff --git a/plugins/meta/site.js b/plugins/meta/site.js
index bd1003934..5a7ed63d9 100644
--- a/plugins/meta/site.js
+++ b/plugins/meta/site.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     lowestPriority: true,
 
diff --git a/plugins/meta/theme-color.js b/plugins/meta/theme-color.js
index f92f68973..fb23c8634 100644
--- a/plugins/meta/theme-color.js
+++ b/plugins/meta/theme-color.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/twitter-description.js b/plugins/meta/twitter-description.js
index ecb434754..aec8d7689 100644
--- a/plugins/meta/twitter-description.js
+++ b/plugins/meta/twitter-description.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(twitter) {
 
diff --git a/plugins/meta/twitter-labels.js b/plugins/meta/twitter-labels.js
index 1ba25f6f1..b38146fc3 100644
--- a/plugins/meta/twitter-labels.js
+++ b/plugins/meta/twitter-labels.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     lowestPriority: true, 
 
diff --git a/plugins/meta/twitter-site.js b/plugins/meta/twitter-site.js
index 57deb18ba..60c331145 100644
--- a/plugins/meta/twitter-site.js
+++ b/plugins/meta/twitter-site.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     lowestPriority: true,
 
diff --git a/plugins/meta/twitter-title.js b/plugins/meta/twitter-title.js
index 44f3ecd53..1db8a7262 100644
--- a/plugins/meta/twitter-title.js
+++ b/plugins/meta/twitter-title.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(twitter) {
 
diff --git a/plugins/meta/video-duration.js b/plugins/meta/video-duration.js
index 6974a47a0..0757023ce 100644
--- a/plugins/meta/video-duration.js
+++ b/plugins/meta/video-duration.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/video.js b/plugins/meta/video.js
index 5267040dd..dc83691d9 100644
--- a/plugins/meta/video.js
+++ b/plugins/meta/video.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
 
diff --git a/plugins/meta/yandex-ovs.js b/plugins/meta/yandex-ovs.js
index 5553bfc9a..034c32f92 100644
--- a/plugins/meta/yandex-ovs.js
+++ b/plugins/meta/yandex-ovs.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: 'ovs',
 

From dcf962351692f9eb7dd91c3a492bb1a54703f454 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 13 Oct 2021 22:51:42 +0300
Subject: [PATCH 003/102] validators

---
 lib/plugins/validators/async/20_checkFavicon.js      | 11 +++++------
 lib/plugins/validators/async/21_checkContentType.js  | 12 ++++++------
 lib/plugins/validators/async/22_imageSize.js         |  8 ++++----
 lib/plugins/validators/html5_multimedia.js           |  2 +-
 lib/plugins/validators/media.js                      |  2 +-
 lib/plugins/validators/player_no_scrolling.js        |  2 +-
 lib/plugins/validators/sync/00_validateLink.js       |  4 ++--
 lib/plugins/validators/sync/01_qa_rels.js            |  4 ++--
 lib/plugins/validators/sync/02_html5_multimedia.js   |  4 ++--
 lib/plugins/validators/sync/03_media.js              |  4 ++--
 lib/plugins/validators/sync/04_ssl_validator.js      |  2 +-
 lib/plugins/validators/sync/05_camoImage.js          |  5 ++---
 lib/plugins/validators/sync/06_autoplay.js           |  4 ++--
 lib/plugins/validators/sync/07_resizable.js          |  6 +++---
 lib/plugins/validators/sync/08_render.js             | 10 +++++-----
 lib/plugins/validators/sync/09_app_hml5.js           |  2 +-
 lib/plugins/validators/sync/10_video_hml5.js         |  2 +-
 lib/plugins/validators/sync/11_image_type_by_ext.js  |  2 +-
 .../validators/sync/12_player_no_scrolling.js        |  4 ++--
 lib/plugins/validators/sync/14_options.js            |  2 +-
 lib/plugins/validators/sync/15_duplicateLink.js      |  4 ++--
 21 files changed, 47 insertions(+), 49 deletions(-)

diff --git a/lib/plugins/validators/async/20_checkFavicon.js b/lib/plugins/validators/async/20_checkFavicon.js
index 6ecce3aa0..768359e37 100644
--- a/lib/plugins/validators/async/20_checkFavicon.js
+++ b/lib/plugins/validators/async/20_checkFavicon.js
@@ -1,10 +1,9 @@
-var urlLib = require('url');
-var _ = require('underscore');
+import * as urlLib from 'url';
+import * as _ from 'underscore';
+import * as utils from '../../../utils.js';
+import * as cache from '../../../cache.js';
 
-var utils = require('../../../utils');
-var cache = require('../../../cache');
-
-module.exports = {
+export default {
 
     startIteration: function(iterationPluginContext) {
         iterationPluginContext.multiCache = new cache.MultiCache();
diff --git a/lib/plugins/validators/async/21_checkContentType.js b/lib/plugins/validators/async/21_checkContentType.js
index 166ca7d90..49058ec5b 100644
--- a/lib/plugins/validators/async/21_checkContentType.js
+++ b/lib/plugins/validators/async/21_checkContentType.js
@@ -1,10 +1,10 @@
-const utils = require('../../../utils');
-const urlLib = require('url');
-const multimedia = require('../html5_multimedia');
-const mediaPlugin = require('../media');
-const player_no_scrolling = require('../player_no_scrolling');
+import * as utils from '../../../utils.js';
+import * as urlLib from 'url';
+import * as multimedia from '../html5_multimedia.js';
+import * as mediaPlugin from '../media.js';
+import * as player_no_scrolling from '../player_no_scrolling.js';
 
-module.exports = {
+export default {
 
     prepareLink: function(url, link, options, pluginContext, cb) {
 
diff --git a/lib/plugins/validators/async/22_imageSize.js b/lib/plugins/validators/async/22_imageSize.js
index 797f3d5ad..3cc66d446 100644
--- a/lib/plugins/validators/async/22_imageSize.js
+++ b/lib/plugins/validators/async/22_imageSize.js
@@ -1,9 +1,9 @@
-const utils = require('../../../utils');
-const cache = require('../../../cache');
-const mediaPlugin = require('../media');
+import * as utils from '../../../utils.js';
+import * as cache from '../../../cache.js';
+import * as mediaPlugin from '../media.js';
 
 
-module.exports = {
+export default {
 
     startIteration: function(iterationPluginContext) {
         iterationPluginContext.multiCache = new cache.MultiCache();
diff --git a/lib/plugins/validators/html5_multimedia.js b/lib/plugins/validators/html5_multimedia.js
index 747d9e41d..08ecc65e0 100644
--- a/lib/plugins/validators/html5_multimedia.js
+++ b/lib/plugins/validators/html5_multimedia.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     prepareLink: function(link) {
 
diff --git a/lib/plugins/validators/media.js b/lib/plugins/validators/media.js
index 985e802b1..3880bde90 100644
--- a/lib/plugins/validators/media.js
+++ b/lib/plugins/validators/media.js
@@ -92,7 +92,7 @@ function moveMediaAttrs(link) {
     }
 }
 
-module.exports = {
+export default {
 
     notPlugin: true,
 
diff --git a/lib/plugins/validators/player_no_scrolling.js b/lib/plugins/validators/player_no_scrolling.js
index fb65a851f..691655d9e 100644
--- a/lib/plugins/validators/player_no_scrolling.js
+++ b/lib/plugins/validators/player_no_scrolling.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     prepareLink: function(link) {
 
diff --git a/lib/plugins/validators/sync/00_validateLink.js b/lib/plugins/validators/sync/00_validateLink.js
index 5db62d447..f49dcfa26 100644
--- a/lib/plugins/validators/sync/00_validateLink.js
+++ b/lib/plugins/validators/sync/00_validateLink.js
@@ -1,6 +1,6 @@
-const urlLib = require('url');
+import * as urlLib from 'url';
 
-module.exports = {
+export default {
 
     prepareLink: function(url, link) {
 
diff --git a/lib/plugins/validators/sync/01_qa_rels.js b/lib/plugins/validators/sync/01_qa_rels.js
index f204d8953..8b9a87424 100644
--- a/lib/plugins/validators/sync/01_qa_rels.js
+++ b/lib/plugins/validators/sync/01_qa_rels.js
@@ -1,7 +1,7 @@
-var pluginLoader = require('../../../loader/pluginLoader'),
+import * as pluginLoader from '../../../loader/pluginLoader.js',
     plugins = pluginLoader._plugins;
 
-module.exports = {
+export default {
 
     prepareLink: function(whitelistRecord, options, link, pluginId) {
 
diff --git a/lib/plugins/validators/sync/02_html5_multimedia.js b/lib/plugins/validators/sync/02_html5_multimedia.js
index dbfb0bafc..ad402ae31 100644
--- a/lib/plugins/validators/sync/02_html5_multimedia.js
+++ b/lib/plugins/validators/sync/02_html5_multimedia.js
@@ -1,6 +1,6 @@
-const multimedia = require('../html5_multimedia');
+import * as multimedia from '../html5_multimedia.js';
 
-module.exports = {
+export default {
 
     prepareLink: function(link, options) {
         multimedia.prepareLink(link, options);
diff --git a/lib/plugins/validators/sync/03_media.js b/lib/plugins/validators/sync/03_media.js
index 37b5e0812..338bd3d66 100644
--- a/lib/plugins/validators/sync/03_media.js
+++ b/lib/plugins/validators/sync/03_media.js
@@ -1,6 +1,6 @@
-var mediaPlugin = require('../media');
+import * as mediaPlugin from '../media.js';
 
-module.exports = {
+export default {
 
     prepareLink: function(link, options) {
 
diff --git a/lib/plugins/validators/sync/04_ssl_validator.js b/lib/plugins/validators/sync/04_ssl_validator.js
index 3821fe267..49d46d51a 100644
--- a/lib/plugins/validators/sync/04_ssl_validator.js
+++ b/lib/plugins/validators/sync/04_ssl_validator.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     prepareLink: function(link, url, options) {
 
diff --git a/lib/plugins/validators/sync/05_camoImage.js b/lib/plugins/validators/sync/05_camoImage.js
index 50db4e887..7bd1c8d31 100644
--- a/lib/plugins/validators/sync/05_camoImage.js
+++ b/lib/plugins/validators/sync/05_camoImage.js
@@ -1,8 +1,7 @@
 /*
 Should be last async plugin to prevent size check over camo.
 */
-
-var crypto = require('crypto');
+import * as crypto from 'crypto';
 
 function isSSL(link) {
     return link.rel.indexOf('ssl') > -1;
@@ -12,7 +11,7 @@ function isImage(link) {
     return link.type.match(/^image/);
 }
 
-module.exports = {
+export default {
 
     // Get from default CONFIG. Not supposed to be enabled by dynamic custom provider options.
     notPlugin: !CONFIG.providerOptions.camoProxy,
diff --git a/lib/plugins/validators/sync/06_autoplay.js b/lib/plugins/validators/sync/06_autoplay.js
index 2015e164f..2733bf16a 100644
--- a/lib/plugins/validators/sync/06_autoplay.js
+++ b/lib/plugins/validators/sync/06_autoplay.js
@@ -1,6 +1,6 @@
-var _ = require('underscore');
+import * as _ from 'underscore';
 
-module.exports = {
+export default {
 
     prepareLink: function(link) {
 
diff --git a/lib/plugins/validators/sync/07_resizable.js b/lib/plugins/validators/sync/07_resizable.js
index 2f1a4f16e..14131b1bf 100644
--- a/lib/plugins/validators/sync/07_resizable.js
+++ b/lib/plugins/validators/sync/07_resizable.js
@@ -1,7 +1,7 @@
-const utils = require('../../../utils');
-const URL = require('url');
+import * as utils from '../../../utils.js';
+import * as URL from 'url';
 
-module.exports = {
+export default {
 
     prepareLink: function(url, options, link) {
 
diff --git a/lib/plugins/validators/sync/08_render.js b/lib/plugins/validators/sync/08_render.js
index 9f985bc4e..5ed5a38d2 100644
--- a/lib/plugins/validators/sync/08_render.js
+++ b/lib/plugins/validators/sync/08_render.js
@@ -1,10 +1,10 @@
-var ejs = require('ejs'),
-    fs = require('fs');
+import * as ejs from 'ejs';
+import * as fs from 'fs';
 
-var pluginLoader = require('../../../loader/pluginLoader'),
-    templates = pluginLoader._templates;
+import * as pluginLoader from '../../../loader/pluginLoader.js';
+var templates = pluginLoader._templates;
 
-module.exports = {
+export default {
     prepareLink: function(link, pluginId) {
 
         if (!link.href && !link.html && (link.template || link.template_context)) {
diff --git a/lib/plugins/validators/sync/09_app_hml5.js b/lib/plugins/validators/sync/09_app_hml5.js
index 3bc64e21e..9ab459855 100644
--- a/lib/plugins/validators/sync/09_app_hml5.js
+++ b/lib/plugins/validators/sync/09_app_hml5.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     prepareLink: function(link) {
         // Add 'html5' rel to 'app'.
diff --git a/lib/plugins/validators/sync/10_video_hml5.js b/lib/plugins/validators/sync/10_video_hml5.js
index 29e4920f8..0386bfea6 100644
--- a/lib/plugins/validators/sync/10_video_hml5.js
+++ b/lib/plugins/validators/sync/10_video_hml5.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     prepareLink: function(link) {
         // Add 'html5' rel to 'video/*'.
diff --git a/lib/plugins/validators/sync/11_image_type_by_ext.js b/lib/plugins/validators/sync/11_image_type_by_ext.js
index 3555038e2..000ec43bc 100644
--- a/lib/plugins/validators/sync/11_image_type_by_ext.js
+++ b/lib/plugins/validators/sync/11_image_type_by_ext.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     prepareLink: function(link) {
 
diff --git a/lib/plugins/validators/sync/12_player_no_scrolling.js b/lib/plugins/validators/sync/12_player_no_scrolling.js
index da8581ade..0ad6db321 100644
--- a/lib/plugins/validators/sync/12_player_no_scrolling.js
+++ b/lib/plugins/validators/sync/12_player_no_scrolling.js
@@ -1,6 +1,6 @@
-const player_no_scrolling = require('../player_no_scrolling');
+import * as player_no_scrolling from '../player_no_scrolling.js';
 
-module.exports = {
+export default {
 
     prepareLink: function(link, options) {
         player_no_scrolling.prepareLink(link, options);
diff --git a/lib/plugins/validators/sync/14_options.js b/lib/plugins/validators/sync/14_options.js
index 680be6adc..175855bfe 100644
--- a/lib/plugins/validators/sync/14_options.js
+++ b/lib/plugins/validators/sync/14_options.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     prepareLink: function(link) {
 
diff --git a/lib/plugins/validators/sync/15_duplicateLink.js b/lib/plugins/validators/sync/15_duplicateLink.js
index af00028b9..2587163d7 100644
--- a/lib/plugins/validators/sync/15_duplicateLink.js
+++ b/lib/plugins/validators/sync/15_duplicateLink.js
@@ -1,4 +1,4 @@
-var _ = require('underscore');
+import * as _ from 'underscore';
 
 function findBestMedia(m1, m2) {
 
@@ -57,7 +57,7 @@ function findBestMedia(m1, m2) {
     return (m1.width > m2.width || m1.height > m2.height) ? m1 : m2;
 }
 
-module.exports = {
+export default {
 
     prepareLink: function(link, pluginContext) {
 

From 6ed00f3af07a896e7226bbddd44a4ca70293465f Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 13 Oct 2021 22:52:13 +0300
Subject: [PATCH 004/102] system plugins

---
 lib/plugins/system/cheerio.js                 |  6 +++---
 lib/plugins/system/decode.js                  |  4 ++--
 .../CollectingHandlerForMutliTarget.js        |  3 +--
 lib/plugins/system/htmlparser/htmlparser.js   | 20 +++++++++----------
 .../system/htmlparser/nonHtmlContentData.js   |  4 ++--
 lib/plugins/system/ld.js                      |  2 +-
 lib/plugins/system/meta/HTMLMetaHandler.js    | 12 +++++------
 lib/plugins/system/meta/cachedMeta.js         | 12 +++++------
 lib/plugins/system/meta/ld-json.js            |  4 ++--
 lib/plugins/system/meta/meta.js               | 16 +++++++--------
 lib/plugins/system/oembed/autoDiscovery.js    |  4 ++--
 lib/plugins/system/oembed/knownEndpoints.js   |  4 ++--
 lib/plugins/system/oembed/oembed.js           | 13 ++++++------
 lib/plugins/system/oembed/oembedUtils.js      | 17 ++++++++--------
 lib/plugins/system/og.js                      |  2 +-
 lib/plugins/system/readability.js             |  4 ++--
 lib/plugins/system/twitter.js                 |  2 +-
 17 files changed, 62 insertions(+), 67 deletions(-)

diff --git a/lib/plugins/system/cheerio.js b/lib/plugins/system/cheerio.js
index a547d5848..3182f848e 100644
--- a/lib/plugins/system/cheerio.js
+++ b/lib/plugins/system/cheerio.js
@@ -1,8 +1,8 @@
-var cheerio = require('cheerio');
-var htmlparser2 = require("htmlparser2");
+import * as cheerio from 'cheerio';
+import * as htmlparser2 from "htmlparser2";
 var DomHandler = htmlparser2.DomHandler;
 
-module.exports = {
+export default {
 
     provides: 'self',
 
diff --git a/lib/plugins/system/decode.js b/lib/plugins/system/decode.js
index 1075cd4bd..e89f9e537 100644
--- a/lib/plugins/system/decode.js
+++ b/lib/plugins/system/decode.js
@@ -1,6 +1,6 @@
-var utils = require('../../utils');
+import * as utils from '../../utils.js';
 
-module.exports = {
+export default {
 
     provides: 'self',
 
diff --git a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
index 0be2fde4c..4956f13fa 100644
--- a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
+++ b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
@@ -9,8 +9,7 @@ function CollectingHandlerForMutliTarget(cbsArray){
     this._cbsArray = cbsArray || [];
     this.events = [];
 }
-
-var EVENTS = require("htmlparser2").EVENTS;
+import { EVENTS } from "htmlparser2";
 
 Object.keys(EVENTS).forEach(function(name) {
 
diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 18f9fd0f3..8c7071579 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -1,17 +1,15 @@
-var _ = require('underscore');
-var urlLib = require('url');
-var htmlparser2 = require('htmlparser2');
+import * as _ from 'underscore';
+import * as urlLib from 'url';
+import * as htmlparser2 from 'htmlparser2';
 var Parser = htmlparser2.Parser;
-
-var cache = require('../../../cache');
-var utils = require('../../../utils');
-var libUtils = require('../../../utils');
-var metaUtils = require('../meta/utils');
+import * as cache from '../../../cache.js';
+import * as utils from '../../../utils.js';
+import * as libUtils from '../../../utils.js';
+import * as metaUtils from '../meta/utils.js';
 var getUrlFunctional = utils.getUrlFunctional;
+import * as CollectingHandlerForMutliTarget from './CollectingHandlerForMutliTarget.js';
 
-var CollectingHandlerForMutliTarget = require('./CollectingHandlerForMutliTarget');
-
-module.exports = {
+export default {
 
     provides: [
         'self',
diff --git a/lib/plugins/system/htmlparser/nonHtmlContentData.js b/lib/plugins/system/htmlparser/nonHtmlContentData.js
index 59623f5cf..9c5e58356 100644
--- a/lib/plugins/system/htmlparser/nonHtmlContentData.js
+++ b/lib/plugins/system/htmlparser/nonHtmlContentData.js
@@ -1,6 +1,6 @@
-var sysUtils = require('../../../../logging');
+import * as sysUtils from '../../../../logging.js';
 
-module.exports = {
+export default {
 
     provides: [
         // Run for all who requests htmlparser or meta.
diff --git a/lib/plugins/system/ld.js b/lib/plugins/system/ld.js
index 320d7b1bb..1b92f1ed0 100644
--- a/lib/plugins/system/ld.js
+++ b/lib/plugins/system/ld.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: 'self',
 
diff --git a/lib/plugins/system/meta/HTMLMetaHandler.js b/lib/plugins/system/meta/HTMLMetaHandler.js
index 1ed92b9ac..7827d5715 100644
--- a/lib/plugins/system/meta/HTMLMetaHandler.js
+++ b/lib/plugins/system/meta/HTMLMetaHandler.js
@@ -1,10 +1,8 @@
-var decodeHTML5 = require('entities').decodeHTML5;
-var _ = require('underscore');
-var url = require('url');
-
-var utils = require('../../../utils');
-
-var ldParser = require('./ld-json');
+import { decodeHTML5 } from 'entities';
+import * as _ from 'underscore';
+import * as url from 'url';
+import * as utils from '../../../utils.js';
+import * as ldParser from './ld-json.js';
 
 var getCharset = utils.getCharset;
 var encodeText = utils.encodeText;
diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js
index 2344d3178..74b9f9200 100644
--- a/lib/plugins/system/meta/cachedMeta.js
+++ b/lib/plugins/system/meta/cachedMeta.js
@@ -1,10 +1,10 @@
-var urlLib = require('url');
-var cache = require('../../../cache');
-var sysUtils = require('../../../../logging');
-var utils = require('./utils');
-var libUtils = require('../../../utils');
+import * as urlLib from 'url';
+import * as cache from '../../../cache.js';
+import * as sysUtils from '../../../../logging.js';
+import * as utils from './utils.js';
+import * as libUtils from '../../../utils.js';
 
-module.exports = {
+export default {
 
     provides: [
         '__noCachedMeta',
diff --git a/lib/plugins/system/meta/ld-json.js b/lib/plugins/system/meta/ld-json.js
index a0e22e2fb..b5d7abe4a 100644
--- a/lib/plugins/system/meta/ld-json.js
+++ b/lib/plugins/system/meta/ld-json.js
@@ -1,5 +1,5 @@
-const sysUtils = require('../../../../logging');
-const utils = require('../../../utils');
+import * as sysUtils from '../../../../logging.js';
+import * as utils from '../../../utils.js';
 
 module.exports = function(result, decode, uri) {
 
diff --git a/lib/plugins/system/meta/meta.js b/lib/plugins/system/meta/meta.js
index ddf76704f..ac72c3524 100644
--- a/lib/plugins/system/meta/meta.js
+++ b/lib/plugins/system/meta/meta.js
@@ -1,11 +1,11 @@
-var HTMLMetaHandler = require('./HTMLMetaHandler');
-var cache = require('../../../cache');
-var sysUtils = require('../../../../logging');
-var libUtils = require('../../../utils');
-var iconv = require('iconv-lite');
-var utils = require('./utils');
-
-module.exports = {
+import * as HTMLMetaHandler from './HTMLMetaHandler.js';
+import * as cache from '../../../cache.js';
+import * as sysUtils from '../../../../logging.js';
+import * as libUtils from '../../../utils.js';
+import * as iconv from 'iconv-lite';
+import * as utils from './utils.js';
+
+export default {
 
     provides: 'self',
 
diff --git a/lib/plugins/system/oembed/autoDiscovery.js b/lib/plugins/system/oembed/autoDiscovery.js
index 50bc9b54c..050963729 100644
--- a/lib/plugins/system/oembed/autoDiscovery.js
+++ b/lib/plugins/system/oembed/autoDiscovery.js
@@ -1,7 +1,7 @@
-const oembedUtils = require('./oembedUtils');
+import * as oembedUtils from './oembedUtils.js';
 const RE = CONFIG.IGNORE_DOMAINS_RE || CONFIG.BLACKLIST_DOMAINS_RE;
 
-module.exports = {
+export default {
 
     provides: 'oembedLinks',
 
diff --git a/lib/plugins/system/oembed/knownEndpoints.js b/lib/plugins/system/oembed/knownEndpoints.js
index a55fdb82b..b767eca7c 100644
--- a/lib/plugins/system/oembed/knownEndpoints.js
+++ b/lib/plugins/system/oembed/knownEndpoints.js
@@ -1,6 +1,6 @@
-var oembedUtils = require('./oembedUtils');
+import * as oembedUtils from './oembedUtils.js';
 
-module.exports = {
+export default {
 
     provides: ['oembedLinks', '__noOembedLinks'],
 
diff --git a/lib/plugins/system/oembed/oembed.js b/lib/plugins/system/oembed/oembed.js
index 8026aca8e..d3b868500 100644
--- a/lib/plugins/system/oembed/oembed.js
+++ b/lib/plugins/system/oembed/oembed.js
@@ -1,9 +1,8 @@
-const oembedUtils = require('./oembedUtils');
-const cheerio = require('cheerio');
-const entities = require('entities');
-
-const URL = require('url');
-const querystring = require('querystring');
+import * as oembedUtils from './oembedUtils.js';
+import * as cheerio from 'cheerio';
+import * as entities from 'entities';
+import * as URL from 'url';
+import * as querystring from 'querystring';
 
 const pxRe = /^([\d\.]+)(?:px)?$/;
 const percentRe = /^([\d\.]+)%$/;
@@ -80,7 +79,7 @@ function getOembedIframeAttr(oembed) {
     };
 }
 
-module.exports = {
+export default {
 
     provides: ['self', 'oembedError'],
 
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index ee9b1389e..6f339a39c 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -1,10 +1,10 @@
-var sax = require('sax');
-var urlLib = require('url');
-var async = require('async');
-
-var utils = require('../../../utils');
-var sysUtils = require('../../../../logging');
-var cache = require('../../../cache');
+import * as sax from 'sax';
+import * as urlLib from 'url';
+import * as async from 'async';
+import * as utils from '../../../utils.js';
+import * as sysUtils from '../../../../logging.js';
+import * as cache from '../../../cache.js';
+import { readFile } from 'fs/promises';
 
 var getUrlFunctional = utils.getUrlFunctional;
 var getCharset = utils.getCharset;
@@ -20,7 +20,8 @@ exports.notPlugin = true;
  * @return {String} The oembed uri
  */
 function lookupStaticProviders(uri) {
-    var providers = require('./providers.json');
+    
+    const providers = JSON.parse(await readFile(new URL('./providers.json', import.meta.url)));
 
     var protocolMatch = uri.match(/^(https?:\/\/)/);
     if (!protocolMatch || /^https?:\/\/blog\./i.test(uri)) {
diff --git a/lib/plugins/system/og.js b/lib/plugins/system/og.js
index 21a24b7ed..abd626c9b 100644
--- a/lib/plugins/system/og.js
+++ b/lib/plugins/system/og.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: 'self',
 
diff --git a/lib/plugins/system/readability.js b/lib/plugins/system/readability.js
index 989ce0da8..fdbd84d7e 100644
--- a/lib/plugins/system/readability.js
+++ b/lib/plugins/system/readability.js
@@ -1,6 +1,6 @@
-var Readability = require('readabilitySAX').Readability;
+import { Readability } from 'readabilitySAX';
 
-module.exports = {
+export default {
 
     provides: 'self',
 
diff --git a/lib/plugins/system/twitter.js b/lib/plugins/system/twitter.js
index 7d5b4e262..4c26e4ad0 100644
--- a/lib/plugins/system/twitter.js
+++ b/lib/plugins/system/twitter.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     provides: 'self',
 

From 2454b14914ff649391e6dcd08de2638680d13201 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 14 Oct 2021 15:33:37 +0300
Subject: [PATCH 005/102] plugins: exports and imports

---
 plugins/custom/domain-icon.js                      | 14 ++++++--------
 plugins/custom/fb-error.js                         |  4 ++--
 plugins/custom/noindex/noindex-header.js           |  4 ++--
 plugins/custom/noindex/noindex-meta.js             |  4 ++--
 plugins/custom/og-image-rel-image.js               |  4 ++--
 plugins/domains/archive.org.js                     |  4 ++--
 plugins/domains/art19.com.js                       |  8 ++++----
 .../brightcove.com/players.brightcove.net.js       |  6 +++---
 plugins/domains/c-span.org/c-span.org-options.js   |  6 ++++--
 plugins/domains/cartodb.com.js                     |  4 ++--
 plugins/domains/codepen.io.js                      |  8 ++++----
 plugins/domains/dailymail.co.uk/dailymail.video.js |  8 +++++---
 plugins/domains/dailymotion.com/dailymotion.com.js |  6 +++---
 plugins/domains/facebook.com/facebook.ld.js        |  6 ++++--
 plugins/domains/facebook.com/facebook.meta.js      |  9 ++++++---
 plugins/domains/facebook.com/facebook.redirects.js |  4 ++--
 plugins/domains/facebook.com/facebook.thumbnail.js |  7 +++++--
 plugins/domains/flickr.com/utils.js                | 14 +++++---------
 plugins/domains/google.com/docs.google.com.js      |  4 ++--
 plugins/domains/google.com/maps.google.com.js      |  8 ++++----
 .../instagram.com/instagram-post-allow-meta.js     |  6 ++++--
 .../instagram.com/instagram-post-rate-limit-429.js |  6 ++++--
 plugins/domains/instagram.com/instagram.com.js     |  6 +++---
 plugins/domains/itunes.apple.com/apple.music.js    |  6 +++---
 plugins/domains/kickstarter.com.js                 |  4 ++--
 plugins/domains/maps.yandex.ru/maps.yandex.ru.js   |  4 ++--
 plugins/domains/mixcloud.com.js                    |  8 ++++----
 plugins/domains/openstreetmap.org.js               |  6 +++---
 plugins/domains/scribd.com/scribd.com-error.js     |  7 ++++---
 plugins/domains/scribd.com/scribd.com.js           | 10 +++++-----
 plugins/domains/slideshare.net.js                  |  6 +++---
 plugins/domains/soundcloud.com/soundcloud.com.js   |  8 ++++----
 plugins/domains/soundcloud.com/w.soundcloud.com.js |  4 ++--
 plugins/domains/speakerdeck.com.js                 |  4 ++--
 plugins/domains/spotify.com-meta-fallback.js       |  6 ++++--
 plugins/domains/strawpoll.me.js                    |  6 +++---
 plugins/domains/ted.com/ted.com.js                 |  4 ++--
 plugins/domains/theatlas.com/theatlas.com.js       |  4 ++--
 plugins/domains/tumblr.com/tumblr.api.js           |  6 +++---
 plugins/domains/tumblr.com/tumblr.oembed.js        |  6 ++++--
 plugins/domains/tumblr.com/tumblr.photo.js         | 10 ++++++----
 plugins/domains/tumblr.com/tumblr.text.js          |  7 ++++---
 plugins/domains/tumblr.com/tumblr.video.js         |  7 ++++---
 plugins/domains/twitch.tv/twitch.tv.js             |  4 ++--
 plugins/domains/twitter.com/twitter.og.js          |  6 ++++--
 plugins/domains/twitter.com/twitter.status.js      | 12 ++++++------
 plugins/domains/twitter.com/twitter.throttle.js    |  7 +++++--
 plugins/domains/usatoday-allowEmbedURL.js          |  6 ++++--
 plugins/domains/v.qq.com.js                        |  4 ++--
 plugins/domains/vimeo.com.js                       |  4 ++--
 plugins/domains/xkcd.com.js                        |  4 ++--
 plugins/domains/youtube.com/youtube.channel.js     |  4 ++--
 plugins/domains/youtube.com/youtube.playlist.js    |  4 ++--
 plugins/domains/youtube.com/youtube.video.js       | 10 +++++-----
 plugins/links/article/article.js                   |  4 ++--
 plugins/links/embedURL/embedURL.js                 |  6 +++---
 plugins/links/embedURL/ld-video.js                 |  4 ++--
 plugins/links/google-docs-viewer.js                |  4 ++--
 plugins/links/hosted/23video-hosted.js             |  4 ++--
 plugins/links/hosted/ooyala.js                     |  4 ++--
 plugins/links/hosted/promo-oembed.js               |  4 ++--
 plugins/links/hosted/promo.js                      |  6 +++---
 plugins/links/iframely-link.js                     |  6 +++---
 plugins/links/image_src.js                         |  4 ++--
 plugins/links/oembed-photo-html.js                 |  4 ++--
 plugins/links/oembed-video.js                      |  6 +++---
 plugins/links/og-image.js                          |  4 ++--
 plugins/links/og-video.js                          |  6 +++---
 plugins/links/prerender/checkAppFlag.js            |  4 ++--
 plugins/links/prerender/prerender.js               |  6 +++---
 plugins/links/prerender/react-app-fb-fallback.js   |  6 +++---
 plugins/links/sailthru.js                          |  4 ++--
 plugins/links/thumbnail.js                         |  4 ++--
 plugins/links/twitter-stream.js                    |  4 ++--
 plugins/links/utils.js                             |  4 ++--
 plugins/meta/description-from-p-tag.js             |  4 ++--
 plugins/meta/ld-article.js                         |  4 ++--
 plugins/meta/ld-product.js                         |  4 ++--
 plugins/meta/media-detector.js                     |  4 ++--
 plugins/meta/noindex-robots.js                     |  4 ++--
 plugins/meta/oembed-description.js                 |  4 ++--
 81 files changed, 243 insertions(+), 217 deletions(-)

diff --git a/plugins/custom/domain-icon.js b/plugins/custom/domain-icon.js
index 1b4b332a7..7dea9e587 100644
--- a/plugins/custom/domain-icon.js
+++ b/plugins/custom/domain-icon.js
@@ -1,13 +1,11 @@
 // use this mixin for domain plugins where you do not want to pull out htmlparser but do need an icon or logo
+import * as core from '../../lib/core.js';
+import * as cache from '../../lib/cache.js';
+import * as async from 'async';
+import * as _ from 'underscore';
+import log from '../../logging.js';
 
-var core = require('../../lib/core');
-var cache = require('../../lib/cache');
-var async = require('async');
-var _ = require('underscore');
-
-var log = exports.log = require('../../logging').log;
-
-module.exports = {
+export default {
 
     provides: 'domain_icons',
 
diff --git a/plugins/custom/fb-error.js b/plugins/custom/fb-error.js
index ea429000e..5d298fe4b 100644
--- a/plugins/custom/fb-error.js
+++ b/plugins/custom/fb-error.js
@@ -1,6 +1,6 @@
-var logging = require('../../logging');
+import * as logging from '../../logging.js';
 
-module.exports = {
+export default {
 
     getLink: function(oembedError, url, cb) {
 
diff --git a/plugins/custom/noindex/noindex-header.js b/plugins/custom/noindex/noindex-header.js
index bab65f360..19944f13c 100644
--- a/plugins/custom/noindex/noindex-header.js
+++ b/plugins/custom/noindex/noindex-header.js
@@ -1,6 +1,6 @@
-var pluginUtils = require('./utils');
+import * as pluginUtils from './utils.js';
 
-module.exports = {
+export default {
 
     getData: function(htmlparser, cb) {
         var headers = htmlparser.headers;
diff --git a/plugins/custom/noindex/noindex-meta.js b/plugins/custom/noindex/noindex-meta.js
index ef08942d2..83ffbd843 100644
--- a/plugins/custom/noindex/noindex-meta.js
+++ b/plugins/custom/noindex/noindex-meta.js
@@ -1,6 +1,6 @@
-var pluginUtils = require('./utils');
+import * as pluginUtils from './utils.js';
 
-module.exports = {
+export default {
 
     getData: function(meta, cb) {
         if (pluginUtils.checkRobots(meta.robots, cb)) {
diff --git a/plugins/custom/og-image-rel-image.js b/plugins/custom/og-image-rel-image.js
index 370877d03..ede29456c 100644
--- a/plugins/custom/og-image-rel-image.js
+++ b/plugins/custom/og-image-rel-image.js
@@ -1,4 +1,4 @@
-var _ = require("underscore");
+import * as _ from "underscore";
 
 var rel = [CONFIG.R.image, CONFIG.R.og];
 
@@ -18,7 +18,7 @@ function getImageLinks(image) {
     }];
 }
 
-module.exports = {
+export default {
 
     getLinks: function(og) {
 
diff --git a/plugins/domains/archive.org.js b/plugins/domains/archive.org.js
index ef7dd4e38..2a19def8c 100644
--- a/plugins/domains/archive.org.js
+++ b/plugins/domains/archive.org.js
@@ -1,6 +1,6 @@
-var utils = require('../../lib/utils');
+import * as utils from '../../lib/utils.js';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/archive\.org\/details\/([^\/]+)\/?\??/i
diff --git a/plugins/domains/art19.com.js b/plugins/domains/art19.com.js
index 96806681b..d9b440fb2 100644
--- a/plugins/domains/art19.com.js
+++ b/plugins/domains/art19.com.js
@@ -1,8 +1,8 @@
-var $ = require('cheerio');
-const querystring = require('querystring');
-const URL = require("url");
+import * as $ from 'cheerio';
+import * as querystring from 'querystring';
+import * as URL from "url";
 
-module.exports = {
+export default {
 
     re: [
         /^(https?:\/\/art19\.com\/shows\/[a-zA-Z0-9\-_]+\/episodes\/[a-zA-Z0-9\-_]+)/i,
diff --git a/plugins/domains/brightcove.com/players.brightcove.net.js b/plugins/domains/brightcove.com/players.brightcove.net.js
index 6ac245852..5d43b6de7 100644
--- a/plugins/domains/brightcove.com/players.brightcove.net.js
+++ b/plugins/domains/brightcove.com/players.brightcove.net.js
@@ -1,7 +1,7 @@
-var cheerio = require('cheerio');
-var utils = require('../../../lib/utils');
+import * as cheerio from 'cheerio';
+import * as utils from '../../../lib/utils.js';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/players\.brightcove\.net\/\d+\/[a-zA-Z0-9]+_[a-zA-Z0-9]+\/index\.html\?videoId=\d+/i
diff --git a/plugins/domains/c-span.org/c-span.org-options.js b/plugins/domains/c-span.org/c-span.org-options.js
index 2eea886d6..13535429f 100644
--- a/plugins/domains/c-span.org/c-span.org-options.js
+++ b/plugins/domains/c-span.org/c-span.org-options.js
@@ -1,6 +1,8 @@
-module.exports = {
+import c_span_org from './c-span.org.js';
 
-    re: require('./c-span.org.js').re,
+export default {
+
+    re: c_span_org.re,
 
     getData: function(url, options) {
         options.exposeStatusCode = true;
diff --git a/plugins/domains/cartodb.com.js b/plugins/domains/cartodb.com.js
index f10d2b3cf..3f2ac1184 100644
--- a/plugins/domains/cartodb.com.js
+++ b/plugins/domains/cartodb.com.js
@@ -1,6 +1,6 @@
-var cheerio = require('cheerio');
+import * as cheerio from 'cheerio';
 
-module.exports = {
+export default {
 
     re: /^https?:(\/\/[\w-]+\.carto(?:db)?\.com\/(?:u\/[\w-]+\/)?viz\/[a-z0-9-]+)/i,
 
diff --git a/plugins/domains/codepen.io.js b/plugins/domains/codepen.io.js
index 7fe20444a..9c8200624 100644
--- a/plugins/domains/codepen.io.js
+++ b/plugins/domains/codepen.io.js
@@ -1,8 +1,8 @@
-const $ = require('cheerio');
-const querystring = require('querystring');
-const URL = require("url");
+import * as $ from 'cheerio';
+import * as querystring from 'querystring';
+import * as URL from "url";
 
-module.exports = {
+export default {
 
     re: /https?:\/\/codepen\.io\/(?:[a-z0-9\-_]+\/)?(pen|details|full)\/([a-z0-9\-]+)/i,
 
diff --git a/plugins/domains/dailymail.co.uk/dailymail.video.js b/plugins/domains/dailymail.co.uk/dailymail.video.js
index dbca24d02..e0f0fd525 100644
--- a/plugins/domains/dailymail.co.uk/dailymail.video.js
+++ b/plugins/domains/dailymail.co.uk/dailymail.video.js
@@ -1,8 +1,10 @@
-const decodeHTML5 = require('entities').decodeHTML5;
+import { decodeHTML5 } from 'entities';
+import dailymail_embeddedvideo from './dailymail.embeddedvideo.js';
+import dailymail_galleryvideo from './dailymail.galleryvideo.js';
 
-module.exports = {
+export default {
 
-    re: [].concat(require('./dailymail.embeddedvideo').re, require('./dailymail.galleryvideo').re),
+    re: [].concat(dailymail_embeddedvideo.re, dailymail_galleryvideo.re),
 
     provides: 'dailymailVideo',
 
diff --git a/plugins/domains/dailymotion.com/dailymotion.com.js b/plugins/domains/dailymotion.com/dailymotion.com.js
index 57a8b5146..b4b6f1afd 100644
--- a/plugins/domains/dailymotion.com/dailymotion.com.js
+++ b/plugins/domains/dailymotion.com/dailymotion.com.js
@@ -1,6 +1,7 @@
-const querystring = require('querystring');
+import * as querystring from 'querystring';
+import * as request from 'request';
 
-module.exports = {
+export default {
 
     mixins: [
         "oembed-title",
@@ -49,7 +50,6 @@ module.exports = {
 
     tests: [{
         getUrls: function(cb) {
-            var request = require('request');
             request({
                 url: 'https://api.dailymotion.com/videos',
                 json: true
diff --git a/plugins/domains/facebook.com/facebook.ld.js b/plugins/domains/facebook.com/facebook.ld.js
index af0d41371..dbec152eb 100644
--- a/plugins/domains/facebook.com/facebook.ld.js
+++ b/plugins/domains/facebook.com/facebook.ld.js
@@ -1,6 +1,8 @@
-module.exports = {
+import facebook_video_re from './facebook.video.js';
 
-    re: require('./facebook.video').re,
+export default {
+
+    re: facebook_video_re.re,
 
     getMeta: function(__allowFBThumbnail, schemaVideoObject) {
 
diff --git a/plugins/domains/facebook.com/facebook.meta.js b/plugins/domains/facebook.com/facebook.meta.js
index bdaa48d0f..160486128 100644
--- a/plugins/domains/facebook.com/facebook.meta.js
+++ b/plugins/domains/facebook.com/facebook.meta.js
@@ -1,6 +1,9 @@
-const entities = require('entities');
+import * as entities from 'entities';
 
-module.exports = {
+import facebook_post from './facebook.post.js';
+import facebook_video from './facebook.video.js';
+
+export default {
 
     /**
      * HEADS-UP: New endpoints as of Oct 24, 2020:
@@ -9,7 +12,7 @@ module.exports = {
      * as desribed on https://github.com/itteco/iframely/issues/284.
      */ 
 
-    re: [].concat(require('./facebook.post').re, require('./facebook.video').re),
+    re: [].concat(facebook_post.re, facebook_video.re),
 
     mixins: [
         "domain-icon",
diff --git a/plugins/domains/facebook.com/facebook.redirects.js b/plugins/domains/facebook.com/facebook.redirects.js
index d24ee0f3d..f531da557 100644
--- a/plugins/domains/facebook.com/facebook.redirects.js
+++ b/plugins/domains/facebook.com/facebook.redirects.js
@@ -1,6 +1,6 @@
-var URL = require("url");
+import * as URL from "url";
 
-module.exports = {
+export default {
 
     re: [        
         /^https?:\/\/m\.facebook\.com\/story\.php/i,
diff --git a/plugins/domains/facebook.com/facebook.thumbnail.js b/plugins/domains/facebook.com/facebook.thumbnail.js
index e16d3ee79..dfa4418b3 100644
--- a/plugins/domains/facebook.com/facebook.thumbnail.js
+++ b/plugins/domains/facebook.com/facebook.thumbnail.js
@@ -1,4 +1,7 @@
-module.exports = {
+import facebook_post from './facebook.post.js';
+import facebook_video from './facebook.video.js';
+
+export default {
 
     provides: "__allowFBThumbnail",
 
@@ -7,7 +10,7 @@ module.exports = {
     // then we grab meta and get og:image from there if it's not "security checked" for rate limits.
     // Similar to what we do in domain-icon: ignore if failed.
 
-    re: [].concat(require('./facebook.post').re, require('./facebook.video').re),
+    re: [].concat(facebook_post.re, facebook_video.re),
 
     getLink: function(url, __allowFBThumbnail, meta) {
 
diff --git a/plugins/domains/flickr.com/utils.js b/plugins/domains/flickr.com/utils.js
index bfd6a708a..7cd7eee1b 100644
--- a/plugins/domains/flickr.com/utils.js
+++ b/plugins/domains/flickr.com/utils.js
@@ -1,12 +1,10 @@
-(function() {
-
-    var async = require('async');
-
     var API_URI = 'https://api.flickr.com/services/rest';
 
-    exports.notPlugin = true;
+    export const notPlugin = true;
 
-    exports.getPhotoSizes = function(photo_id, request, api_key, cb) {
+    // TODO: is this used anywhere?
+
+    export function getPhotoSizes(photo_id, request, api_key, cb) {
         request({
                 uri: API_URI,
                 qs: {
@@ -22,6 +20,4 @@
                     cb(error || (body && body.message), body && body.sizes && body.sizes.size);
                 }
             }, cb);
-    };
-
-})();
\ No newline at end of file
+    };
\ No newline at end of file
diff --git a/plugins/domains/google.com/docs.google.com.js b/plugins/domains/google.com/docs.google.com.js
index 12f82d77b..67f89853d 100644
--- a/plugins/domains/google.com/docs.google.com.js
+++ b/plugins/domains/google.com/docs.google.com.js
@@ -1,6 +1,6 @@
-var decodeHTML5 = require('entities').decodeHTML5;
+import { decodeHTML5 } from 'entities';
 
-module.exports = {
+export default {
 
     provides: "schemaFileObject",
 
diff --git a/plugins/domains/google.com/maps.google.com.js b/plugins/domains/google.com/maps.google.com.js
index 6be27faef..4bf5c73c4 100644
--- a/plugins/domains/google.com/maps.google.com.js
+++ b/plugins/domains/google.com/maps.google.com.js
@@ -1,6 +1,6 @@
-var URL = require("url");
-var _ = require('underscore');
-var QueryString = require("querystring");
+import * as URL from "url";
+import * as _ from 'underscore';
+import * as QueryString from "querystring";
 
 var TypeMap = {
     m: 'roadmap',
@@ -14,7 +14,7 @@ function diameterToZoom (diameter) {
     return zoom < 0 ? 0 : zoom > 20 ? 20 : zoom;
 }
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/maps\.google\.(?:com?\.)?[a-z]+\/(?:maps(?:\/ms|\/preview)?)?[\?\#].+/i,
diff --git a/plugins/domains/instagram.com/instagram-post-allow-meta.js b/plugins/domains/instagram.com/instagram-post-allow-meta.js
index 662027d18..87f7dfc19 100644
--- a/plugins/domains/instagram.com/instagram-post-allow-meta.js
+++ b/plugins/domains/instagram.com/instagram-post-allow-meta.js
@@ -1,6 +1,8 @@
-module.exports = {
+import instagram_com from './instagram.com.js';
 
-    re: require('./instagram.com').re,
+export default {
+
+    re: instagram_com.re,
 
     provides: 'ipOG',
 
diff --git a/plugins/domains/instagram.com/instagram-post-rate-limit-429.js b/plugins/domains/instagram.com/instagram-post-rate-limit-429.js
index 003fca3ba..1c904d1a4 100644
--- a/plugins/domains/instagram.com/instagram-post-rate-limit-429.js
+++ b/plugins/domains/instagram.com/instagram-post-rate-limit-429.js
@@ -1,6 +1,8 @@
-module.exports = {
+import instagram_com from './instagram.com.js';
 
-    re: require('./instagram.com').re,
+export default {
+
+    re: instagram_com.re,
 
     provides: 'ipOG',
 
diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js
index f6c041973..734fdda2b 100644
--- a/plugins/domains/instagram.com/instagram.com.js
+++ b/plugins/domains/instagram.com/instagram.com.js
@@ -1,7 +1,7 @@
-const cheerio = require('cheerio');
-const decodeHTML5 = require('entities').decodeHTML5;
+import * as cheerio from 'cheerio';
+import { decodeHTML5 as decodeHTML5 } from 'entities';
 
-module.exports = {
+export default {
 
     /**
      * HEADS-UP: New endpoints as of Oct 24, 2020:
diff --git a/plugins/domains/itunes.apple.com/apple.music.js b/plugins/domains/itunes.apple.com/apple.music.js
index 37417f587..87aab4d33 100644
--- a/plugins/domains/itunes.apple.com/apple.music.js
+++ b/plugins/domains/itunes.apple.com/apple.music.js
@@ -1,7 +1,7 @@
-const URL = require('url');
-const _ = require('underscore');
+import * as URL from 'url';
+import * as _ from 'underscore';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/music\.apple\.com\/(\w{2})\/(album)(?:\/[^\/]+)?\/id(\d+)\?i=(\d+)?/i,
diff --git a/plugins/domains/kickstarter.com.js b/plugins/domains/kickstarter.com.js
index 5958ec936..61ab356ec 100644
--- a/plugins/domains/kickstarter.com.js
+++ b/plugins/domains/kickstarter.com.js
@@ -1,6 +1,6 @@
-var utils = require('../../lib/utils');
+import * as utils from '../../lib/utils.js';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)?kickstarter\.com\/projects\/[a-zA-Z0-9-]+\/[a-zA-Z0-9-]+\/?(?:widget\/video\.html)?(?:\?.*)?$/i
diff --git a/plugins/domains/maps.yandex.ru/maps.yandex.ru.js b/plugins/domains/maps.yandex.ru/maps.yandex.ru.js
index 345e0ee47..0c781433f 100644
--- a/plugins/domains/maps.yandex.ru/maps.yandex.ru.js
+++ b/plugins/domains/maps.yandex.ru/maps.yandex.ru.js
@@ -1,6 +1,6 @@
-var URL = require("url");
+import * as URL from "url";
 
-module.exports = {
+export default {
 
     re: /^https:\/\/yandex\.ru\/maps\//,
 
diff --git a/plugins/domains/mixcloud.com.js b/plugins/domains/mixcloud.com.js
index 565ad6302..986327001 100644
--- a/plugins/domains/mixcloud.com.js
+++ b/plugins/domains/mixcloud.com.js
@@ -1,8 +1,8 @@
-const cheerio = require('cheerio');
-const querystring = require('querystring');
-const URL = require("url");
+import * as cheerio from 'cheerio';
+import * as querystring from 'querystring';
+import * as URL from "url";
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)mixcloud\.com\/(?:live\/)?[a-zA-Z0-9\.\-_]+\/?(?:\?.+)?$/,
diff --git a/plugins/domains/openstreetmap.org.js b/plugins/domains/openstreetmap.org.js
index 43cb9f47a..59a6f9aef 100644
--- a/plugins/domains/openstreetmap.org.js
+++ b/plugins/domains/openstreetmap.org.js
@@ -1,5 +1,5 @@
-var URL = require("url");
-var QueryString = require("querystring");
+import * as URL from "url";
+import * as QueryString from "querystring";
 
 var LayerMap = {
     M: 'mapnik',
@@ -53,7 +53,7 @@ function getBBox(lat, lon, zoom, width, height) {
     return [lonlat_s[0], lonlat_s[1], lonlat_e[0], lonlat_e[1]];
 }
 
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:www\.)?openstreetmap\.org\/(?:node\/\d+)?(?:\?.+|\#.*map=.+|export\/embed\.html\?)/i,
 
diff --git a/plugins/domains/scribd.com/scribd.com-error.js b/plugins/domains/scribd.com/scribd.com-error.js
index 74ae24214..e155a89e7 100644
--- a/plugins/domains/scribd.com/scribd.com-error.js
+++ b/plugins/domains/scribd.com/scribd.com-error.js
@@ -1,8 +1,9 @@
-const URL = require("url");
+import * as URL from "url";
+import scribd_com from './scribd.com.js';
 
-module.exports = {
+export default {
 
-    re: require('./scribd.com').re,
+    re: scribd_com.re,
 
     provides: ["scribdData"],
 
diff --git a/plugins/domains/scribd.com/scribd.com.js b/plugins/domains/scribd.com/scribd.com.js
index dffc16ab7..33084c603 100644
--- a/plugins/domains/scribd.com/scribd.com.js
+++ b/plugins/domains/scribd.com/scribd.com.js
@@ -1,9 +1,9 @@
-const $ = require('cheerio');
-const utils = require('../../../lib/utils');
-const querystring = require('querystring');
-const URL = require("url");
+import * as $ from 'cheerio';
+import * as utils from '../../../lib/utils.js';
+import * as querystring from 'querystring';
+import * as URL from "url";
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www|\w{2})\.scribd\.com\/(doc|document|embeds|presentation|fullscreen)\/(\d+)/i
diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js
index bfde2a8be..4961fe614 100644
--- a/plugins/domains/slideshare.net.js
+++ b/plugins/domains/slideshare.net.js
@@ -1,7 +1,7 @@
-var utils = require('../../lib/utils');
-var $ = require('cheerio');
+import * as utils from '../../lib/utils.js';
+import * as $ from 'cheerio';
 
-module.exports = {
+export default {
 
     mixins: [
         // "*" // Linking to * will enable oembed-rich and will result in incorrect aspect-ratios
diff --git a/plugins/domains/soundcloud.com/soundcloud.com.js b/plugins/domains/soundcloud.com/soundcloud.com.js
index c6d11a2e6..88f1196f6 100644
--- a/plugins/domains/soundcloud.com/soundcloud.com.js
+++ b/plugins/domains/soundcloud.com/soundcloud.com.js
@@ -1,8 +1,8 @@
-const $ = require('cheerio');
-const querystring = require('querystring');
-const URL = require("url");
+import * as $ from 'cheerio';
+import * as querystring from 'querystring';
+import * as URL from "url";
 
-module.exports = {
+export default {
 
     provides: ['__allow_soundcloud_meta', 'sound', 'iframe'],
 
diff --git a/plugins/domains/soundcloud.com/w.soundcloud.com.js b/plugins/domains/soundcloud.com/w.soundcloud.com.js
index bd8851bae..0cbb93820 100644
--- a/plugins/domains/soundcloud.com/w.soundcloud.com.js
+++ b/plugins/domains/soundcloud.com/w.soundcloud.com.js
@@ -1,6 +1,6 @@
-const URL = require('url');
+import * as URL from 'url';
 
-module.exports = {
+export default {
 
     re: [
         /^https:?\/\/w\.soundcloud\.com\/player\/?\?/i
diff --git a/plugins/domains/speakerdeck.com.js b/plugins/domains/speakerdeck.com.js
index a36ffb5ea..53a2b417f 100644
--- a/plugins/domains/speakerdeck.com.js
+++ b/plugins/domains/speakerdeck.com.js
@@ -1,6 +1,6 @@
-var $ = require('cheerio');
+import * as $ from 'cheerio';
 
-module.exports = {
+export default {
 
     mixins: [
         "oembed-title",
diff --git a/plugins/domains/spotify.com-meta-fallback.js b/plugins/domains/spotify.com-meta-fallback.js
index cad168e23..b445f041f 100644
--- a/plugins/domains/spotify.com-meta-fallback.js
+++ b/plugins/domains/spotify.com-meta-fallback.js
@@ -1,6 +1,8 @@
-module.exports = {
+import spotify_com from './spotify.com.js';
 
-    re: require('./spotify.com').re,
+export default {
+
+    re: spotify_com.re,
 
     provides: ['meta'],
 
diff --git a/plugins/domains/strawpoll.me.js b/plugins/domains/strawpoll.me.js
index 20066396f..9de15417b 100644
--- a/plugins/domains/strawpoll.me.js
+++ b/plugins/domains/strawpoll.me.js
@@ -1,7 +1,7 @@
-var $ = require('cheerio');
-var entities = require('entities');
+import * as $ from 'cheerio';
+import * as entities from 'entities';
 
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:www\.)?strawpoll\.me\/([0-9]+)$/i,
 
diff --git a/plugins/domains/ted.com/ted.com.js b/plugins/domains/ted.com/ted.com.js
index 3d88243c7..a36b6dab7 100644
--- a/plugins/domains/ted.com/ted.com.js
+++ b/plugins/domains/ted.com/ted.com.js
@@ -1,6 +1,6 @@
-const URL = require('url');
+import * as URL from 'url';
 
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:www\.)?ted\.com\/talks\//i,
 
diff --git a/plugins/domains/theatlas.com/theatlas.com.js b/plugins/domains/theatlas.com/theatlas.com.js
index db2f7ff29..04dd29855 100644
--- a/plugins/domains/theatlas.com/theatlas.com.js
+++ b/plugins/domains/theatlas.com/theatlas.com.js
@@ -1,6 +1,6 @@
-var utils = require('../../../lib/utils');
+import * as utils from '../../../lib/utils.js';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)?theatlas\.com\/charts\/([a-zA-Z0-9]+)/i              
diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js
index 0cbdd25d3..aeaa5e58b 100644
--- a/plugins/domains/tumblr.com/tumblr.api.js
+++ b/plugins/domains/tumblr.com/tumblr.api.js
@@ -1,7 +1,7 @@
-var $ = require('cheerio');
-var _ = require('underscore');
+import * as $ from 'cheerio';
+import * as _ from 'underscore';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/([a-z0-9-]+\.tumblr\.com)\/(post|image)\/(\d+)(?:\/[a-z0-9-]+)?/i,
diff --git a/plugins/domains/tumblr.com/tumblr.oembed.js b/plugins/domains/tumblr.com/tumblr.oembed.js
index 08c85c763..640647f80 100644
--- a/plugins/domains/tumblr.com/tumblr.oembed.js
+++ b/plugins/domains/tumblr.com/tumblr.oembed.js
@@ -1,8 +1,10 @@
 /* Tumblr embed codes are broken as of Feb 13, 2020 */
 
-module.exports = {
+import tumblr_api from './tumblr.api.js';
 
-    re: require('./tumblr.api').re,
+export default {
+
+    re: tumblr_api.re,
 
     getLink: function(tumblr_post, oembed, options) {
 
diff --git a/plugins/domains/tumblr.com/tumblr.photo.js b/plugins/domains/tumblr.com/tumblr.photo.js
index d4eefd06d..2aa51a1e8 100644
--- a/plugins/domains/tumblr.com/tumblr.photo.js
+++ b/plugins/domains/tumblr.com/tumblr.photo.js
@@ -1,9 +1,11 @@
-var _ = require('underscore');
-var $ = require('cheerio');
+import * as _ from 'underscore';
+import * as $ from 'cheerio';
 
-module.exports = {
+import tumblr_api from './tumblr.api.js';
 
-    re: require('./tumblr.api').re,    
+export default {
+
+    re: tumblr_api.re,
 
     getLinks: function(tumblr_post) {
 
diff --git a/plugins/domains/tumblr.com/tumblr.text.js b/plugins/domains/tumblr.com/tumblr.text.js
index aae3a86ef..196c2e180 100644
--- a/plugins/domains/tumblr.com/tumblr.text.js
+++ b/plugins/domains/tumblr.com/tumblr.text.js
@@ -1,8 +1,9 @@
-var $ = require('cheerio');
+import * as $ from 'cheerio';
+import tumblr_api from './tumblr.api.js';
 
-module.exports = {
+export default {
 
-    re: require('./tumblr.api').re,
+    re: tumblr_api.re,
 
     getMeta: function (tumblr_post) {
         if (tumblr_post.type == "text") {
diff --git a/plugins/domains/tumblr.com/tumblr.video.js b/plugins/domains/tumblr.com/tumblr.video.js
index 8fca974f5..a00d5ce01 100644
--- a/plugins/domains/tumblr.com/tumblr.video.js
+++ b/plugins/domains/tumblr.com/tumblr.video.js
@@ -1,8 +1,9 @@
-const $ = require('cheerio');
+import * as $ from 'cheerio';
+import tumblr_api from './tumblr.api.js';
 
-module.exports = {
+export default {
 
-    re: require('./tumblr.api').re, 
+    re: tumblr_api.re, 
 
     getLink: function(tumblr_post) {
 
diff --git a/plugins/domains/twitch.tv/twitch.tv.js b/plugins/domains/twitch.tv/twitch.tv.js
index ddb773282..dec8a5eb6 100644
--- a/plugins/domains/twitch.tv/twitch.tv.js
+++ b/plugins/domains/twitch.tv/twitch.tv.js
@@ -1,6 +1,6 @@
-const URL = require('url');
+import * as URL from 'url';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.|go\.)?twitch\.tv\/[a-zA-Z0-9_]+\/v\/(\d+)/i,
diff --git a/plugins/domains/twitter.com/twitter.og.js b/plugins/domains/twitter.com/twitter.og.js
index 990e7732d..bdd394715 100644
--- a/plugins/domains/twitter.com/twitter.og.js
+++ b/plugins/domains/twitter.com/twitter.og.js
@@ -1,6 +1,8 @@
-module.exports = {
+import twitter_status from './twitter.status.js';
 
-    re: require('./twitter.status').re,
+export default {
+
+    re: twitter_status.re,
 
     provides: ['twitter_og'],
 
diff --git a/plugins/domains/twitter.com/twitter.status.js b/plugins/domains/twitter.com/twitter.status.js
index acb3ffd20..a759ecd18 100644
--- a/plugins/domains/twitter.com/twitter.status.js
+++ b/plugins/domains/twitter.com/twitter.status.js
@@ -1,10 +1,10 @@
-var async = require('async');
-var cache = require('../../../lib/cache');
-var sysUtils = require('../../../logging');
-var _ = require('underscore');
-var entities = require('entities');
+import * as async from 'async';
+import * as cache from '../../../lib/cache.js';
+import * as sysUtils from '../../../logging.js';
+import * as _ from 'underscore';
+import * as entities from 'entities';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/twitter\.com\/(?:\w+)\/status(?:es)?\/(\d+)/i
diff --git a/plugins/domains/twitter.com/twitter.throttle.js b/plugins/domains/twitter.com/twitter.throttle.js
index 2e5ee6ca2..70d49b683 100644
--- a/plugins/domains/twitter.com/twitter.throttle.js
+++ b/plugins/domains/twitter.com/twitter.throttle.js
@@ -1,6 +1,9 @@
-module.exports = {
+import twitter_timelines from './twitter.timelines.js';
+import twitter_status from './twitter.status.js';
 
-    re: require('./twitter.timelines').re.concat(require('./twitter.status').re),
+export default {
+
+    re: twitter_timelines.re.concat(twitter_status.re),
 
     provides: ["meta"],
 
diff --git a/plugins/domains/usatoday-allowEmbedURL.js b/plugins/domains/usatoday-allowEmbedURL.js
index 9692ecfcb..a3528d11e 100644
--- a/plugins/domains/usatoday-allowEmbedURL.js
+++ b/plugins/domains/usatoday-allowEmbedURL.js
@@ -1,6 +1,8 @@
-module.exports = {
+import usatoday_com from './usatoday.com.js';
 
-    re: require('./usatoday.com').re,
+export default {
+
+    re: usatoday_com.re,
 
     provides: '__allowEmbedURL',
 
diff --git a/plugins/domains/v.qq.com.js b/plugins/domains/v.qq.com.js
index f56561d83..920b079e3 100644
--- a/plugins/domains/v.qq.com.js
+++ b/plugins/domains/v.qq.com.js
@@ -1,4 +1,4 @@
-var _ = require('underscore');
+import * as _ from 'underscore';
 
 var res = [
     /^https?:\/\/v\.qq\.com\/page\/\w\/\w\/\w\/(\w+)\.html$/i,
@@ -8,7 +8,7 @@ var res = [
     /^https?:\/\/v\.qq\.com\/\w\/cover\/\w+\/(\w+)\.html/i
 ];
 
-module.exports = {
+export default {
 
     re: res,
 
diff --git a/plugins/domains/vimeo.com.js b/plugins/domains/vimeo.com.js
index cad8493a5..47a0320ac 100644
--- a/plugins/domains/vimeo.com.js
+++ b/plugins/domains/vimeo.com.js
@@ -1,6 +1,6 @@
-const querystring = require('querystring');
+import * as querystring from 'querystring';
 
-module.exports = {
+export default {
 
     re: [
         /^https:\/\/vimeo\.com(?:\/channels?\/\w+)?\/\d+/i, // Includes private reviews like /video/123/ABC.
diff --git a/plugins/domains/xkcd.com.js b/plugins/domains/xkcd.com.js
index a65114092..be1f2cbc3 100644
--- a/plugins/domains/xkcd.com.js
+++ b/plugins/domains/xkcd.com.js
@@ -1,6 +1,6 @@
-const decodeHTML5 = require('entities').decodeHTML5;
+import { decodeHTML5 as decodeHTML5 } from 'entities';
 
-module.exports = {
+export default {
 
     re: /^https?:\/\/(?:www.)?xkcd\.com\/\d+/i,
 
diff --git a/plugins/domains/youtube.com/youtube.channel.js b/plugins/domains/youtube.com/youtube.channel.js
index c72cfee42..18702364f 100644
--- a/plugins/domains/youtube.com/youtube.channel.js
+++ b/plugins/domains/youtube.com/youtube.channel.js
@@ -1,6 +1,6 @@
-const sysUtils = require('../../../logging')
+import * as sysUtils from '../../../logging.js'
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)?youtube\.com\/(channel)\/([a-zA-Z0-9_-]+)/i,
diff --git a/plugins/domains/youtube.com/youtube.playlist.js b/plugins/domains/youtube.com/youtube.playlist.js
index 2e6cdde9d..d1bf1c0ec 100644
--- a/plugins/domains/youtube.com/youtube.playlist.js
+++ b/plugins/domains/youtube.com/youtube.playlist.js
@@ -1,6 +1,6 @@
-const querystring = require('querystring');
+import * as querystring from 'querystring';
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/www\.youtube\.com\/playlist\?(?:[=\-_a-zA-Z0-9&]+)?list=([\-_a-zA-Z0-9]+)/i
diff --git a/plugins/domains/youtube.com/youtube.video.js b/plugins/domains/youtube.com/youtube.video.js
index d380453a1..6487a1201 100644
--- a/plugins/domains/youtube.com/youtube.video.js
+++ b/plugins/domains/youtube.com/youtube.video.js
@@ -1,9 +1,9 @@
-const cheerio = require('cheerio');
-const querystring = require('querystring');
-const _ = require('underscore');
-const sysUtils = require('../../../logging')
+import * as cheerio from 'cheerio';
+import * as querystring from 'querystring';
+import * as _ from 'underscore';
+import * as sysUtils from '../../../logging.js'
 
-module.exports = {
+export default {
 
     re: [
         /^https?:\/\/(?:www\.)?youtube\.com\/(?:tv#\/)?watch\/?\?(?:[^&]+&)*v=([a-zA-Z0-9_-]+)/i,
diff --git a/plugins/links/article/article.js b/plugins/links/article/article.js
index 72821e6d0..d603f9154 100644
--- a/plugins/links/article/article.js
+++ b/plugins/links/article/article.js
@@ -1,6 +1,6 @@
-var utils = require('../../../lib/utils');
+import * as utils from '../../../lib/utils.js';
 
-module.exports = {
+export default {
 
     getData: function(readability, meta, __is_general_article) {
 
diff --git a/plugins/links/embedURL/embedURL.js b/plugins/links/embedURL/embedURL.js
index 48c747dfc..6020bbe90 100644
--- a/plugins/links/embedURL/embedURL.js
+++ b/plugins/links/embedURL/embedURL.js
@@ -1,7 +1,7 @@
-const decodeHTML5 = require('entities').decodeHTML5;
-const utils = require('../../../lib/utils');
+import { decodeHTML5 as decodeHTML5 } from 'entities';
+import * as utils from '../../../lib/utils.js';
 
-module.exports = {
+export default {
 
     provides: 'schemaVideoObject',
 
diff --git a/plugins/links/embedURL/ld-video.js b/plugins/links/embedURL/ld-video.js
index a97fd2d69..a076f50b7 100644
--- a/plugins/links/embedURL/ld-video.js
+++ b/plugins/links/embedURL/ld-video.js
@@ -1,6 +1,6 @@
-const cheerio = require('cheerio');
+import * as cheerio from 'cheerio';
 
-module.exports = {
+export default {
 
     provides: [
         'schemaVideoObject', 
diff --git a/plugins/links/google-docs-viewer.js b/plugins/links/google-docs-viewer.js
index 17ea34970..e6215fcfb 100644
--- a/plugins/links/google-docs-viewer.js
+++ b/plugins/links/google-docs-viewer.js
@@ -1,6 +1,6 @@
-const utils = require('../../lib/utils');
+import * as utils from '../../lib/utils.js';
 
-module.exports = {
+export default {
 
     getLink: function(url, __nonHtmlContentData, options) {
 
diff --git a/plugins/links/hosted/23video-hosted.js b/plugins/links/hosted/23video-hosted.js
index 8a163a59f..f5a160504 100644
--- a/plugins/links/hosted/23video-hosted.js
+++ b/plugins/links/hosted/23video-hosted.js
@@ -1,6 +1,6 @@
-var cheerio = require('cheerio');
+import * as cheerio from 'cheerio';
 
-module.exports = {
+export default {
 
     // linked to oembed, so won't run for all URLs
     getLink: function(meta, oembed, whitelistRecord) {
diff --git a/plugins/links/hosted/ooyala.js b/plugins/links/hosted/ooyala.js
index 53f747ec1..852fe93c6 100644
--- a/plugins/links/hosted/ooyala.js
+++ b/plugins/links/hosted/ooyala.js
@@ -1,6 +1,6 @@
-var URL = require('url');
+import * as URL from 'url';
 
-module.exports = {
+export default {
 
     provides: '__ooyalaPlayer',
 
diff --git a/plugins/links/hosted/promo-oembed.js b/plugins/links/hosted/promo-oembed.js
index 034683eb8..5ffbbf58c 100644
--- a/plugins/links/hosted/promo-oembed.js
+++ b/plugins/links/hosted/promo-oembed.js
@@ -1,6 +1,6 @@
-var urlLib = require('url');
+import * as urlLib from 'url';
 
-module.exports = {
+export default {
 
     provides: '__promoUri',
 
diff --git a/plugins/links/hosted/promo.js b/plugins/links/hosted/promo.js
index 979f4d028..9c9bc3aa5 100644
--- a/plugins/links/hosted/promo.js
+++ b/plugins/links/hosted/promo.js
@@ -1,7 +1,7 @@
-var core = require('../../../lib/core');
-var _ = require('underscore');
+import * as core from '../../../lib/core.js';
+import * as _ from 'underscore';
 
-module.exports = {
+export default {
 
     provides: 'self',
 
diff --git a/plugins/links/iframely-link.js b/plugins/links/iframely-link.js
index d826b773f..fd2798050 100644
--- a/plugins/links/iframely-link.js
+++ b/plugins/links/iframely-link.js
@@ -1,7 +1,7 @@
-var utils = require('./utils');
-var _ = require('underscore');
+import * as utils from './utils.js';
+import * as _ from 'underscore';
 
-module.exports = {
+export default {
 
     getLinks: function(meta, whitelistRecord) {
         return _.flatten(_.keys(meta).map(function(key) {
diff --git a/plugins/links/image_src.js b/plugins/links/image_src.js
index 971d0d72b..2655c9145 100644
--- a/plugins/links/image_src.js
+++ b/plugins/links/image_src.js
@@ -1,6 +1,6 @@
-var utils = require('./utils');
+import * as utils from './utils.js';
 
-module.exports = {
+export default {
 
     getLink: function(meta) {
         return utils.getImageLink('image_src', meta);
diff --git a/plugins/links/oembed-photo-html.js b/plugins/links/oembed-photo-html.js
index 54e227957..d3aee0865 100644
--- a/plugins/links/oembed-photo-html.js
+++ b/plugins/links/oembed-photo-html.js
@@ -1,6 +1,6 @@
-var cheerio = require('cheerio');
+import * as cheerio from 'cheerio';
 
-module.exports = {
+export default {
 
     // this is the case of oembed photo or image, but with the html field
     // ex: 
diff --git a/plugins/links/oembed-video.js b/plugins/links/oembed-video.js
index ef5f180e8..bacbe24b3 100644
--- a/plugins/links/oembed-video.js
+++ b/plugins/links/oembed-video.js
@@ -1,7 +1,7 @@
-var cheerio = require('cheerio');
-var entities = require('entities');
+import * as cheerio from 'cheerio';
+import * as entities from 'entities';
 
-module.exports = {
+export default {
 
     getLink: function(oembed, whitelistRecord, url) {
 
diff --git a/plugins/links/og-image.js b/plugins/links/og-image.js
index 599bd7276..f7c9ab9c5 100644
--- a/plugins/links/og-image.js
+++ b/plugins/links/og-image.js
@@ -1,4 +1,4 @@
-var _ = require("underscore");
+import * as _ from "underscore";
 
 var rel = [CONFIG.R.thumbnail, CONFIG.R.og];
 
@@ -25,7 +25,7 @@ function getImageLinks(image) {
     return images;
 }
 
-module.exports = {
+export default {
 
     getLinks: function(og) {
 
diff --git a/plugins/links/og-video.js b/plugins/links/og-video.js
index a7ea76e8c..5e23eabe2 100644
--- a/plugins/links/og-video.js
+++ b/plugins/links/og-video.js
@@ -1,5 +1,5 @@
-var _ = require("underscore");
-var utils = require('./utils');
+import * as _ from "underscore";
+import * as utils from './utils.js';
 
 function getVideoLinks(video, whitelistRecord) {
     
@@ -48,7 +48,7 @@ function getVideoLinks(video, whitelistRecord) {
     return players;
 }
 
-module.exports = {
+export default {
 
     getLinks: function(og, whitelistRecord) {
 
diff --git a/plugins/links/prerender/checkAppFlag.js b/plugins/links/prerender/checkAppFlag.js
index 10377b008..cc35d48c3 100644
--- a/plugins/links/prerender/checkAppFlag.js
+++ b/plugins/links/prerender/checkAppFlag.js
@@ -1,6 +1,6 @@
-const utils = require('./utils');
+import * as utils from './utils.js';
 
-module.exports = {
+export default {
 
     provides: '__appFlag',
 
diff --git a/plugins/links/prerender/prerender.js b/plugins/links/prerender/prerender.js
index a4a432661..d1472cc38 100644
--- a/plugins/links/prerender/prerender.js
+++ b/plugins/links/prerender/prerender.js
@@ -1,7 +1,7 @@
-const core = require('../../../lib/core');
-const utils = require('./utils');
+import * as core from '../../../lib/core.js';
+import * as utils from './utils.js';
 
-module.exports = {
+export default {
 
     highestPriority: true,
 
diff --git a/plugins/links/prerender/react-app-fb-fallback.js b/plugins/links/prerender/react-app-fb-fallback.js
index af3b039e3..78acfc681 100644
--- a/plugins/links/prerender/react-app-fb-fallback.js
+++ b/plugins/links/prerender/react-app-fb-fallback.js
@@ -1,7 +1,7 @@
-const core = require('../../../lib/core');
-const utils = require('./utils');
+import * as core from '../../../lib/core.js';
+import * as utils from './utils.js';
 
-module.exports = {
+export default {
 
     highestPriority: true,
 
diff --git a/plugins/links/sailthru.js b/plugins/links/sailthru.js
index e5f026502..77e381b85 100644
--- a/plugins/links/sailthru.js
+++ b/plugins/links/sailthru.js
@@ -1,6 +1,6 @@
-var utils = require('./utils');
+import * as utils from './utils.js';
 
-module.exports = {
+export default {
 
     provides: 'sailthru',
 
diff --git a/plugins/links/thumbnail.js b/plugins/links/thumbnail.js
index 086087f64..4793e73d3 100644
--- a/plugins/links/thumbnail.js
+++ b/plugins/links/thumbnail.js
@@ -1,6 +1,6 @@
-var utils = require('./utils');
+import * as utils from './utils.js';
 
-module.exports = {
+export default {
 
     getLink: function(meta) {
         if (!meta.og || !meta.og.image) {
diff --git a/plugins/links/twitter-stream.js b/plugins/links/twitter-stream.js
index 82c4a1b2b..bd9922f66 100644
--- a/plugins/links/twitter-stream.js
+++ b/plugins/links/twitter-stream.js
@@ -1,4 +1,4 @@
-var _ = require("underscore");
+import * as _ from "underscore";
 
 function getStreamLinks(twitter, stream, whitelistRecord) {
 
@@ -31,7 +31,7 @@ function getStreamLinks(twitter, stream, whitelistRecord) {
     return player;
 }
 
-module.exports = {
+export default {
 
     getLink: function(twitter, whitelistRecord) {
 
diff --git a/plugins/links/utils.js b/plugins/links/utils.js
index 21be78d8f..b4ce60a4e 100644
--- a/plugins/links/utils.js
+++ b/plugins/links/utils.js
@@ -1,4 +1,4 @@
-var _ = require('underscore');
+import * as _ from 'underscore';
 
 var ALLOWED_TYPES = {};
 
@@ -6,7 +6,7 @@ _.values(CONFIG.T).forEach(function(v) {
     ALLOWED_TYPES[v] = true;
 });
 
-module.exports = {
+export default {
 
     notPlugin: true,
 
diff --git a/plugins/meta/description-from-p-tag.js b/plugins/meta/description-from-p-tag.js
index ace4bbf34..7a4659590 100644
--- a/plugins/meta/description-from-p-tag.js
+++ b/plugins/meta/description-from-p-tag.js
@@ -1,6 +1,6 @@
-var decodeHTML5 = require('entities').decodeHTML5;
+import { decodeHTML5 as decodeHTML5 } from 'entities';
 
-module.exports = {
+export default {
 
     getMeta: function(cheerio, decode, __allowPTagDescription) {
         // Get the text from the first <p> tag that's not in a header
diff --git a/plugins/meta/ld-article.js b/plugins/meta/ld-article.js
index 07e17902c..70441e90a 100644
--- a/plugins/meta/ld-article.js
+++ b/plugins/meta/ld-article.js
@@ -1,6 +1,6 @@
-const cheerio = require('cheerio');
+import * as cheerio from 'cheerio';
 
-module.exports = {
+export default {
 
     getMeta: function(ld) {
         function clean(field) {
diff --git a/plugins/meta/ld-product.js b/plugins/meta/ld-product.js
index 8928aaedd..5aa6b1d1e 100644
--- a/plugins/meta/ld-product.js
+++ b/plugins/meta/ld-product.js
@@ -1,6 +1,6 @@
-const cheerio = require('cheerio');
+import * as cheerio from 'cheerio';
 
-module.exports = {
+export default {
 
     highestPriority: true,
 
diff --git a/plugins/meta/media-detector.js b/plugins/meta/media-detector.js
index f465005bd..8b1e9990e 100644
--- a/plugins/meta/media-detector.js
+++ b/plugins/meta/media-detector.js
@@ -1,6 +1,6 @@
-const utils = require('../links/utils');
+import * as utils from '../links/utils.js';
 
-module.exports = {
+export default {
 
     lowestPriority: true,
 
diff --git a/plugins/meta/noindex-robots.js b/plugins/meta/noindex-robots.js
index 8f40e86d7..a96a997ae 100644
--- a/plugins/meta/noindex-robots.js
+++ b/plugins/meta/noindex-robots.js
@@ -1,6 +1,6 @@
-var oembedUtils = require('../../lib/plugins/system/oembed/oembedUtils');
+import * as oembedUtils from '../../lib/plugins/system/oembed/oembedUtils.js';
 
-module.exports = {
+export default {
 
     getData: function(url, meta, __noOembedLinks, cb) {
 
diff --git a/plugins/meta/oembed-description.js b/plugins/meta/oembed-description.js
index 39cfd66d8..2f9b531df 100644
--- a/plugins/meta/oembed-description.js
+++ b/plugins/meta/oembed-description.js
@@ -1,6 +1,6 @@
-var cheerio = require('cheerio');
+import * as cheerio from 'cheerio';
 
-module.exports = {
+export default {
 
     getMeta: function(oembed) {
         if (oembed.description) {

From 29bc74235e3f5fe9f98a34b16d629a53b87e98e1 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 14 Oct 2021 15:59:21 +0300
Subject: [PATCH 006/102] core: update with ES6 export import

---
 app.js                          |  19 ++--
 cluster.js                      |   6 +-
 config.js                       |  35 +++----
 docs/WRITE-A-PLUGIN.md          |   6 +-
 lib/agent-http2.js              |  36 ++------
 lib/cache-engines/memcached.js  |  12 +--
 lib/cache-engines/node-cache.js |  14 ++-
 lib/cache-engines/redis.js      |   7 +-
 lib/cache.js                    |  29 +++---
 lib/core.js                     |  32 +++----
 lib/html-utils.js               |  21 ++---
 lib/loader/pluginLoader.js      | 118 +++++++++++++-----------
 lib/loader/utils.js             |  16 ++--
 lib/oembed.js                   |   7 +-
 lib/request.js                  |  15 ++-
 lib/utils.js                    |  88 +++++++++---------
 lib/whitelist.js                |  38 ++++----
 logging.js                      |   4 +-
 modules/api/utils.js            |   4 +-
 modules/api/views.js            |  30 +++---
 modules/debug/views.js          |   4 +-
 modules/tests-ui/models.js      |   9 +-
 modules/tests-ui/tester.js      |  22 ++---
 modules/tests-ui/utils.js       |  24 ++---
 modules/tests-ui/views.js       |  22 ++---
 package.json                    |   2 +
 server.js                       |   4 +-
 utils.js                        |  43 ++++-----
 yarn.lock                       | 158 +++++++++++++++++++++++++++-----
 29 files changed, 451 insertions(+), 374 deletions(-)

diff --git a/app.js b/app.js
index 08a9bf4fc..77c03a36d 100644
--- a/app.js
+++ b/app.js
@@ -1,4 +1,4 @@
-var sysUtils = require('./utils');
+import * as sysUtils from './utils.js';
 
 console.log("");
 console.log("Starting Iframely...");
@@ -8,9 +8,9 @@ if (!CONFIG.baseAppUrl) {
   console.warn('Warning: CONFIG.baseAppUrl not set, default value used');
 }
 
-var path = require('path');
-var express = require('express');
-var jsonxml = require('jsontoxml');
+import * as path from 'path';
+import * as express from 'express';
+import * as jsonxml from 'jsontoxml';
 
 var NotFound = sysUtils.NotFound;
 
@@ -42,10 +42,15 @@ app.use(function(req, res, next) {
 
 app.use(sysUtils.cacheMiddleware);
 
+import apiViews from './modules/api/views.js';
+import debugViews from './modules/debug/views.js';
+apiViews(app);
+debugViews(app);
 
-require('./modules/api/views')(app);
-require('./modules/debug/views')(app);
-require('./modules/tests-ui/views')(app);
+if (CONFIG.tests) {
+  const testViews = await import('./modules/tests-ui/views.js');
+  testViews.default(app);
+}
 
 app.use(logErrors);
 app.use(errorHandler);
diff --git a/cluster.js b/cluster.js
index 1f2ccc73d..1c7f393ea 100644
--- a/cluster.js
+++ b/cluster.js
@@ -1,5 +1,5 @@
-var GracefulCluster = require('graceful-cluster').GracefulCluster;
-var sysUtils = require('./utils');
+import { GracefulCluster } from 'graceful-cluster';
+import * as sysUtils from './utils.js';
 
 process.title = 'iframely-cluster';
 
@@ -10,6 +10,6 @@ GracefulCluster.start({
     restartOnTimeout: CONFIG.CLUSTER_WORKER_RESTART_ON_PERIOD,
     restartOnMemory: CONFIG.CLUSTER_WORKER_RESTART_ON_MEMORY_USED,
     serverFunction: function() {
-        require('./server');
+        import('./server');
     }
 });
\ No newline at end of file
diff --git a/config.js b/config.js
index 2eae9a083..6da7fe620 100644
--- a/config.js
+++ b/config.js
@@ -1,17 +1,16 @@
-(function() {
+    import * as _ from 'underscore';
+    import * as path from 'path';
+    import * as fs from 'fs';
 
-    // Monkey patch before you require http for the first time.
-    var majorVersion = process.version.match(/v(\d+)\./);
-    majorVersion = parseInt(majorVersion);
-    if (majorVersion < 10) {
-        process.binding('http_parser').HTTPParser = require('http-parser-js').HTTPParser;
-    }
-
-    var _ = require('underscore');
-    var path = require('path');
-    var fs = require('fs');
+    import { fileURLToPath } from 'url';
+    import { dirname } from 'path';
 
-    var version = require('./package.json').version;
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    
+    import { readFile } from 'fs/promises';
+    const json = JSON.parse(await readFile(new URL('./package.json', import.meta.url)));
+    var version = json.version;
 
     var config = {
 
@@ -382,7 +381,8 @@
     // Providers config loader.
     var local_config_path = path.resolve(__dirname, "config.providers.js");
     if (fs.existsSync(local_config_path)) {
-        var local = require(local_config_path);
+        var local = await import(local_config_path);
+        local = local && local.default;
         _.extend(config, local);
     }
 
@@ -396,11 +396,13 @@
 
     // Try config by NODE_ENV.
     if (fs.existsSync(env_config_path)) {
-        var local = require(env_config_path);
+        var local = await import(env_config_path);
+        local = local && local.default;
 
     } else if (fs.existsSync(local_config_path)) {
         // Else - try local config.
-        var local = require(local_config_path);
+        var local = await import(local_config_path);
+        local = local && local.default;
     }
 
     _.extend(config, local);
@@ -428,5 +430,4 @@
         config.HTTP2_RETRY_CODES[item] = 1;
     });
 
-    module.exports = config;
-})();
+    export default config;
diff --git a/docs/WRITE-A-PLUGIN.md b/docs/WRITE-A-PLUGIN.md
index e00c903e6..55e4b1c7e 100644
--- a/docs/WRITE-A-PLUGIN.md
+++ b/docs/WRITE-A-PLUGIN.md
@@ -125,7 +125,7 @@ See available attributes names to check if similar name exists at [/meta-mapping
 
 See example [/generic/meta/video.js](https://github.com/itteco/iframely/blob/master/plugins/generic/meta/video.js):
 
-    module.exports = {
+    export default {
         getMeta: function(meta) {
 
             // This prevents non useful errors loging with "undefined".
@@ -150,13 +150,13 @@ But `og:title` usually better and contains only article title without site name.
 
 If you want to mark you plugin as worst source of meta (like html `<title>` tag), use `lowestPriority: true`:
 
-    module.exports = {
+    export default {
         lowestPriority: true
     }
 
 If you want to mark your plugin as good source of meta (like og:title), use `highestPriority: true`:
 
-    module.exports = {
+    export default {
         highestPriority: true
     }
 
diff --git a/lib/agent-http2.js b/lib/agent-http2.js
index 3fb7ada1a..ccc2df57b 100644
--- a/lib/agent-http2.js
+++ b/lib/agent-http2.js
@@ -1,25 +1,12 @@
-var semver = require('semver');
-var https = require('https');
-var http2;
+import * as semver from 'semver';
+import * as https from 'https';
 
-if (semver.gte(process.version, '8.13.0')) {
-
-    http2 = require('http2');
-
-} else {
-
-    // Old node without stable http2 support: fallback to https agent. 
-    exports.Http2Agent = https.Agent;
-    return;
-}
-
-var stream = require('stream');
-var util = require('util');
-var http = require('http');
-var tls = require('tls');
-var EventEmitter = require('events').EventEmitter;
-
-var sysUtils = require('../logging');
+import * as http2 from 'http2';
+import * as stream from 'stream';
+import * as util from 'util';
+import * as http from 'http';
+import * as tls from 'tls';
+import { EventEmitter } from 'events';
 
 // Http2Response Class.
 
@@ -91,7 +78,7 @@ Http2Response.prototype._transform = function (chunk, enc, cb) {
 
 // Http2Agent Class
 
-function Http2Agent(options) {
+export function Http2Agent(options) {
 
     if (!(this instanceof Http2Agent))
         return new Http2Agent(options);
@@ -380,10 +367,6 @@ Http2Agent.prototype.createConnection = function(options, tlsSocketCb, http2Sess
     });
 };
 
-exports.Http2Agent = Http2Agent;
-
-
-
 // HTTPS agents. Copy from https.js.
 
 const { inherits } = util;
@@ -425,4 +408,3 @@ function CustomHttpsAgent(options) {
 }
 inherits(CustomHttpsAgent, https.Agent);
 CustomHttpsAgent.prototype.createConnection = createConnection;
-  
\ No newline at end of file
diff --git a/lib/cache-engines/memcached.js b/lib/cache-engines/memcached.js
index d0cce3c47..1868c3f94 100644
--- a/lib/cache-engines/memcached.js
+++ b/lib/cache-engines/memcached.js
@@ -1,10 +1,7 @@
 (function(engine) {
-
-    var sysUtils = require('../../logging');
-
-    var crypto = require('crypto');
-
-    var Memcached = require('memcached');
+import * as sysUtils from '../../logging.js';
+import * as crypto from 'crypto';
+import * as Memcached from 'memcached';
     var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options);
 
     var timeout = CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options && CONFIG.MEMCACHED_OPTIONS.options.timeout;
@@ -16,8 +13,7 @@
     }
 
     function _findKeyMeta(k) {
-
-        var _ = require('underscore');
+import * as _ from 'underscore';
 
         var sk = safeKey(k);
 
diff --git a/lib/cache-engines/node-cache.js b/lib/cache-engines/node-cache.js
index 48227c3ee..ea109dc10 100644
--- a/lib/cache-engines/node-cache.js
+++ b/lib/cache-engines/node-cache.js
@@ -1,16 +1,16 @@
-(function(cache) {
+    import NodeCache from "node-cache";
+    import CONFIG from '../../config.js';
 
-    var NodeCache = require("node-cache");
     var nodeCache = new NodeCache({
         stdTTL: CONFIG.CACHE_TTL,
         checkperiod: CONFIG.CACHE_TTL / 2
     });
 
-    cache.set = function(key, data, options) {
+    export function set(key, data, options) {
         nodeCache.set(key, data, options && options.ttl || CONFIG.CACHE_TTL);
-    };
+    }
 
-    cache.get = function(key, cb) {
+    export function get(key, cb) {
         nodeCache.get(key, function(error, data) {
             if (error) {
                 return cb(null, null);
@@ -22,6 +22,4 @@
                 cb(null, null);
             }
         });
-    };
-
-})(exports);
\ No newline at end of file
+    }
\ No newline at end of file
diff --git a/lib/cache-engines/redis.js b/lib/cache-engines/redis.js
index 451515885..33ffc6c2a 100644
--- a/lib/cache-engines/redis.js
+++ b/lib/cache-engines/redis.js
@@ -1,15 +1,14 @@
 (function(engine) {
-
-    var sysUtils = require('../../logging');
+import * as sysUtils from '../../logging.js';
 
     var client;
 
     if (CONFIG.REDIS_MODE === 'cluster') {
-        var RedisClustr = require('redis-clustr');
+import * as RedisClustr from 'redis-clustr';
 
         client = new RedisClustr(CONFIG.REDIS_CLUSTER_OPTIONS);
     } else {
-        var redis = require('redis');
+import * as redis from 'redis';
         client = redis.createClient(CONFIG.REDIS_OPTIONS);
     }
 
diff --git a/lib/cache.js b/lib/cache.js
index 5169cb7fe..04bbcfdec 100644
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -1,24 +1,25 @@
-(function(cache) {
-
-    var _ = require('underscore');
+    import * as _ from 'underscore';
+    import CONFIG from '../config.js';
 
     var DEFAULT_CACHE = "node-cache";
 
-    function setCachingEngine(path) {
+    export const cache = {};
+
+    async function setCachingEngine(path) {
         try {
 
             if (path.indexOf('/') === -1) {
                 // Not full path.
-                path = './cache-engines/' + path;
+                path = './cache-engines/' + path + '.js';
             }
 
             var id = path.split('/').slice(-1)[0].replace(/\.js$/, '');
 
-            var cache_engine = require(path);
+            var cache_engine = await import(path);
 
             if (!cache_engine.set || !cache_engine.get) {
                 console.warn("Default cache engine used. No get and set methods in cache engine", id);
-                return setCachingEngine(DEFAULT_CACHE);
+                return await setCachingEngine(DEFAULT_CACHE);
             }
 
             cache.set = function(key, data, options) {
@@ -38,18 +39,18 @@
             console.log("Using cache engine:", id);
 
         } catch (ex) {
-            if (path == DEFAULT_CACHE) {
+            if (path.indexOf(DEFAULT_CACHE) > -1) {
                 throw ex;
             } else {
                 console.warn("Default cache engine used. Check CONFIG.CACHE_ENGINE. ", ex.stack);
-                setCachingEngine(DEFAULT_CACHE);
+                await setCachingEngine(DEFAULT_CACHE);
             }
         }
     }
 
-    setCachingEngine(CONFIG.CACHE_ENGINE || DEFAULT_CACHE);
+    await setCachingEngine(CONFIG.CACHE_ENGINE || DEFAULT_CACHE);
 
-    cache.withCache = function(key, func, options, callback) {
+    export function withCache(key, func, options, callback) {
 
         if (typeof options === 'function') {
             callback = options;
@@ -142,7 +143,7 @@
     *
     * */
 
-    var MultiCache = cache.MultiCache = function(options) {
+    export function MultiCache(options) {
         this.options = options;
 
         this.keysDict = {};
@@ -285,6 +286,4 @@
         }
 
         cache.set(this.multiKey, data, this.options);
-    };
-
-})(exports);
\ No newline at end of file
+    };
\ No newline at end of file
diff --git a/lib/core.js b/lib/core.js
index e206c1b72..e216b1d9f 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -1,15 +1,13 @@
-(function(core) {
 
-    var _ = require('underscore'),
-        request = require('request'),
-        urlLib = require('url');
-
-    var pluginUtils = require('./loader/utils'),
-        utils = require('./utils'),
-        sysUtils = require('../logging'),
-        oembedUtils = require('./oembed'),
-        pluginLoader = require('./loader/pluginLoader'),
-        requestWrapper = require('./request');
+    import * as _ from 'underscore';
+    import * as request from 'request';
+    import * as urlLib from 'url';
+    import * as pluginUtils from './loader/utils.js';
+    import * as utils from './utils.js';
+    import * as sysUtils from '../logging.js';
+    import * as oembedUtils from './oembed.js';
+    import * as pluginLoader from './loader/pluginLoader.js';
+    import * as requestWrapper from './request.js';
 
     var plugins = pluginLoader._plugins,
         providedParamsDict = pluginLoader._providedParamsDict,
@@ -1157,7 +1155,7 @@
         }
 
         // Sort links.
-        core.sortLinks(links);
+        sortLinks(links);
 
         var allData = result.allData;
         if (allData) {
@@ -1183,7 +1181,7 @@
         }
     }
 
-    core.sortLinks = function(links) {
+    export function sortLinks(links) {
 
         // Sort links in order of REL according to CONFIG.REL_GROUPS.
         function getRelIndex(rel) {
@@ -1392,7 +1390,7 @@
     /*
     * Run plugins to collect all possible data.
     * */
-    var run = core.run = function(uri, options, cb) {
+    export function run(uri, options, cb) {
 
         if (typeof options === 'function') {
             cb = options;
@@ -1689,13 +1687,13 @@
         asyncMethodCb('initial');
     };
 
-    exports.getPluginData = function(uri, param, getWhitelistRecord, cb) {
+    export function getPluginData(uri, param, getWhitelistRecord, cb) {
         run(uri, {
             fetchParam: param,
             getWhitelistRecord: getWhitelistRecord
         }, cb);
     };
 
-    exports.getOembed = oembedUtils.getOembed;
+    var getOembed = oembedUtils.getOembed;
 
-})(exports);
+    export { getOembed };
diff --git a/lib/html-utils.js b/lib/html-utils.js
index 578ac295c..0107ca9cd 100644
--- a/lib/html-utils.js
+++ b/lib/html-utils.js
@@ -1,7 +1,6 @@
-(function() {
-
-    var $ = require('cheerio');
-    var _ = require('underscore');
+    import * as $ from 'cheerio';
+    import * as _ from 'underscore';
+    import CONFIG from '../config.js';
 
     var defaultPaddingBottom = 100 / CONFIG.DEFAULT_ASPECT_RATIO;
 
@@ -330,7 +329,7 @@
         }
     };
 
-    exports.generateElementWrapperHtml = function(element, link, options) {
+    export function generateElementWrapperHtml(element, link, options) {
 
         if (typeof element === 'string') {
             element = $(element);
@@ -340,7 +339,7 @@
         return $('<div>').append($el).html();
     };
 
-    exports.generateLinkElementHtml = function(link, options) {
+    export function generateLinkElementHtml(link, options) {
         var $el = generateLinkElement(link, options);
         if (_.isString($el)) {
             return $el;
@@ -394,7 +393,7 @@
         return [];
     };
 
-    exports.filterLinksByRel = function(rel, links, options) {
+    export function filterLinksByRel(rel, links, options) {
 
         var options = options || {};
 
@@ -547,7 +546,7 @@
 
     // Good link selection.
 
-    var sortLinks = exports.sortLinks = function(links, autoplay_first, horizontal_first, mp4_first) {
+    export function sortLinks(links, autoplay_first, horizontal_first, mp4_first) {
 
         var options = {
             autoplay_first: autoplay_first, 
@@ -676,7 +675,7 @@
         });
     }
 
-    exports.findMainLink = function(iframely_data, options) {
+    export function findMainLink(iframely_data, options) {
 
         var selectedLink;
 
@@ -702,6 +701,4 @@
         }
 
         return selectedLink;
-    };
-
-})();
\ No newline at end of file
+    };
\ No newline at end of file
diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js
index 0859149c1..8c53d3501 100644
--- a/lib/loader/pluginLoader.js
+++ b/lib/loader/pluginLoader.js
@@ -1,15 +1,11 @@
-(function(pluginLoader) {
 
-    var fs = require('fs'),
-        path = require('path'),
-        _ = require('underscore'),
-        async = require('async'),
-        url = require('url');
-
-    var node_jslint = require('jslint'),
-        JSLINT = node_jslint.load('latest');
-
-    var utils = require('./utils');
+    import * as fs from 'fs';
+    import * as path from 'path';
+    import * as _ from 'underscore';
+    import * as node_jslint from 'jslint';
+    import { readFile } from 'fs/promises';
+    var JSLINT = node_jslint.load('latest');
+    import * as utils from './utils.js';
 
     var PLUGIN_METHODS = utils.PLUGIN_METHODS,
         PLUGINS_FIELDS = utils.PLUGIN_FIELDS,
@@ -17,15 +13,25 @@
         POST_PLUGIN_DEFAULT_PARAMS = utils.POST_PLUGIN_DEFAULT_PARAMS,
         DEFAULT_PARAMS = utils.DEFAULT_PARAMS;
 
-    var plugins = pluginLoader._plugins = {},
-        providedParamsDict = pluginLoader._providedParamsDict = {},
-        usedParamsDict = pluginLoader._usedParamsDict = {},
-        templates = pluginLoader._templates = {},
+    
+    const plugins = {},
+        providedParamsDict = {},
+        usedParamsDict = {},
+        templates = {},
         metaMappings = {},
-        pluginsList = pluginLoader._pluginsList = [],
-        postPluginsList = pluginLoader._postPluginsList = [],
+        pluginsList = [],
+        postPluginsList = [],
         pluginsByDomain = {};
 
+    export {
+        plugins as _plugins,
+        providedParamsDict as _p_providedParamsDictlugins,
+        usedParamsDict as _usedParamsDict,
+        templates as _templates,
+        pluginsList as _pluginsList,
+        postPluginsList as _postPluginsList
+    };
+
     /*
      * ===================
      * Loading plugins
@@ -134,7 +140,7 @@
         return methods;
     }
 
-    function loadPluginFile(pluginPath) {
+    async function loadPluginFile(pluginPath) {
 
         var bits = pluginPath.split(path.sep);
 
@@ -142,16 +148,25 @@
 
             var plugin;
             var pluginDeclaration = {};
+            var notPlugin = false;
 
             // Load plugin.
             try {
-                plugin = require(pluginPath);
+                plugin = await import(pluginPath);
+                if (plugin.notPlugin) {
+                    notPlugin = true;
+                }
+                plugin = plugin && plugin.default;
             } catch(ex) {
                 console.error("Error loading plugin", pluginPath, ex);
                 return;
             }
 
-            if (plugin.notPlugin) {
+            if (!plugin) {
+                console.error("Error loading plugin", pluginPath);
+            }
+
+            if (notPlugin || plugin.notPlugin) {
                 // Skip utils modules.
                 return;
             }
@@ -339,36 +354,36 @@
         return modified;
     }
 
-    function loadPluginDir(pluginPath) {
+    async function loadPluginDir(pluginPath) {
 
         // Scan plugin dir.
         var plugins = fs.readdirSync(pluginPath);
 
-        plugins.forEach(function(plugin_name) {
+        for (const plugin_name of plugins) {
             var plugin = path.resolve(pluginPath, plugin_name);
             var stats = fs.statSync(plugin);
 
             if (stats.isFile()) {
-                loadPluginFile(plugin);
+                await loadPluginFile(plugin);
             }
-        });
+        };
     }
 
-    function scanAllPluginsDir(modulePluginsPath) {
+    async function scanAllPluginsDir(modulePluginsPath) {
 
         // Scan mudule plugins.
         var plugins = fs.readdirSync(modulePluginsPath);
 
-        plugins.forEach(function(plugin_name) {
+        for (const plugin_name of plugins) {
             var plugin = path.resolve(modulePluginsPath, plugin_name);
             var stats = fs.statSync(plugin);
 
             if (stats.isFile()) {
-                loadPluginFile(plugin);
+                await loadPluginFile(plugin);
             } if (stats.isDirectory()) {
-                loadPluginDir(plugin);
+                await loadPluginDir(plugin);
             }
-        });
+        };
     }
 
     function extractDomain(uri) {
@@ -406,7 +421,7 @@
         return patterns;
     }
 
-    pluginLoader.findDomainPlugin = function(uri) {
+    export function findDomainPlugin(uri) {
         var patterns = extractDomainPatterns(uri);
 
         var record, i = 0;
@@ -418,7 +433,7 @@
         return record;
     };
 
-    function scanModulesForPlugins() {
+    async function scanModulesForPlugins() {
 
         // Scan node_modules dir.
         var modulesRootPath = path.resolve('node_modules');
@@ -426,7 +441,7 @@
 
         modules_listing.push(path.resolve('.'));
 
-        modules_listing.forEach(function(modulePath) {
+        for (const modulePath of modules_listing) {
 
             var modulePackagePath = path.resolve(modulePath, 'package.json');
 
@@ -434,43 +449,45 @@
 
                 // Scan plugins.
 
-                var moduleInfo = require(modulePackagePath);
+                
+                const moduleInfo = JSON.parse(await readFile(new URL(modulePackagePath, import.meta.url)));
+
                 if (!moduleInfo["iframely-proxy-plugins"]) {
-                    return;
+                    continue;
                 }
 
-                function loadPlugins() {
+                async function loadPlugins() {
                     var bits = Array.prototype.slice.call(arguments, 0);
                     bits.splice(0, 0, modulePath);
                     var modulePluginsPath = path.resolve.apply(path.resolve, bits);
                     if (fs.existsSync(modulePluginsPath)) {
-                        scanAllPluginsDir(modulePluginsPath);
+                        await scanAllPluginsDir(modulePluginsPath);
                     }
                 }
 
-                loadPlugins('plugins', 'domains');
-                loadPlugins('plugins', 'custom');
-                loadPlugins('plugins', 'links');
-                loadPlugins('plugins', 'meta');
-                loadPlugins('plugins', 'templates');
+                await loadPlugins('plugins', 'domains');
+                await loadPlugins('plugins', 'custom');
+                await loadPlugins('plugins', 'links');
+                await loadPlugins('plugins', 'meta');
+                await loadPlugins('plugins', 'templates');
 
                 // TODO: if has multiple modules_listing - CUSTOM_PLUGINS_PATH will be loaded multiple times.
                 if (CONFIG.CUSTOM_PLUGINS_PATH) {
                     if (fs.existsSync(CONFIG.CUSTOM_PLUGINS_PATH)) {
-                        loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'domains');
-                        loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom');
-                        loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links');
-                        loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta');
-                        loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates');
+                        await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'domains');
+                        await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'custom');
+                        await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'links');
+                        await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'meta');
+                        await loadPlugins(CONFIG.CUSTOM_PLUGINS_PATH, 'templates');
                     } else {
                         console.warn('Custom plugin folder "' + CONFIG.CUSTOM_PLUGINS_PATH + '" not found.');
                     }
                 }
 
-                loadPlugins('lib', 'plugins', 'system');
-                loadPlugins('lib', 'plugins', 'validators');
+                await loadPlugins('lib', 'plugins', 'system');
+                await loadPlugins('lib', 'plugins', 'validators');
             }
-        });
+        }
 
         validateMixins();
         validateDependencies();
@@ -549,6 +566,5 @@
         });
     }
 
-    scanModulesForPlugins();
+    await scanModulesForPlugins();
 
-})(exports);
\ No newline at end of file
diff --git a/lib/loader/utils.js b/lib/loader/utils.js
index d8c1d8ef8..1a465a3b3 100644
--- a/lib/loader/utils.js
+++ b/lib/loader/utils.js
@@ -1,7 +1,5 @@
-(function(utils) {
-
     // TODO: not used anywhere.
-    utils.DEFAULT_PARAMS = [
+    export const DEFAULT_PARAMS = [
         "url",
         "urlMatch",
         "cb",
@@ -11,14 +9,14 @@
         "__readabilityEnabled"
     ];
 
-    utils.POST_PLUGIN_DEFAULT_PARAMS = [
+    export const POST_PLUGIN_DEFAULT_PARAMS = [
         "link",
         "pluginContext",
         "pluginId",
         "iterationPluginContext"
     ];
 
-    utils.PLUGIN_FIELDS_TYPE_DICT = {
+    export const PLUGIN_FIELDS_TYPE_DICT = {
         "getLink": Function,
         "getLinks": Function,
         "getData": Function,
@@ -26,7 +24,7 @@
         "mixins": Array
     };
 
-    utils.PLUGIN_METHODS = [
+    export const PLUGIN_METHODS = [
         "getLink",
         "getLinks",
         "getData",
@@ -34,8 +32,6 @@
         "prepareLink"
     ];
 
-    utils.PLUGIN_FIELDS = utils.PLUGIN_METHODS.concat([
+    export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([
         "mixins"
-    ]);
-
-})(exports);
\ No newline at end of file
+    ]);
\ No newline at end of file
diff --git a/lib/oembed.js b/lib/oembed.js
index 60b74f540..5c2c31d75 100644
--- a/lib/oembed.js
+++ b/lib/oembed.js
@@ -1,8 +1,7 @@
-var _ = require('underscore');
+import * as _ from 'underscore';
+import * as htmlUtils from './html-utils.js';
 
-var htmlUtils = require('./html-utils');
-
-exports.getOembed = function(uri, data, options) {
+export function getOembed(uri, data, options) {
 
     if (!data) {
         return;
diff --git a/lib/request.js b/lib/request.js
index 4b6a1c096..41d88a1eb 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -1,10 +1,9 @@
-var request = require('request');
-var _ = require('underscore');
-var crypto = require('crypto');
-var cache = require('./cache');
-var utils = require('./utils');
-
-var sysUtils = require('../logging');
+import * as request from 'request';
+import * as _ from 'underscore';
+import * as crypto from 'crypto';
+import * as cache from './cache.js';
+import * as utils from './utils.js';
+import * as sysUtils from '../logging.js';
 
 var hash = function(value) {
     return '"' + crypto.createHash('md5').update(value).digest("hex") + '"';
@@ -28,7 +27,7 @@ var hash = function(value) {
 * - callback - final callback will called with data received from `prepareResult`.
 *
 * */
-module.exports = function(options, iframely_options, callback) {
+export default function(options, iframely_options, callback) {
 
     options = _.extend({}, options);
 
diff --git a/lib/utils.js b/lib/utils.js
index a7907b5e6..db115fea2 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,27 +1,27 @@
-var events = require('events');
-var request = require('request');
-var fetch = require('node-fetch');
-const AbortController = require('abort-controller');
-var Http2Agent = require('./agent-http2').Http2Agent;
-var zlib = require('zlib');
-var decompressStream = require('iltorb').decompressStream;
-var iconv = require('iconv-lite');
-var async = require('async');
-var imagesize = require('probe-image-size');
-var _ = require('underscore');
-
-var parseIsoDuration = require('parse-iso-duration');
-var entities = require('entities');
-
-var cache = require('./cache');
-var htmlUtils = require('./html-utils');
-
-var sysUtils = require('../logging');
+import * as events from 'events';
+import * as request from 'request';
+import * as fetch from 'node-fetch';
+import * as AbortController from 'abort-controller';
+import { Http2Agent } from './agent-http2.js';
+import * as zlib from 'zlib';
+import iltorb from 'iltorb';
+var decompressStream = iltorb;
+import * as iconv from 'iconv-lite';
+import * as async from 'async';
+import * as imagesize from 'probe-image-size';
+import * as _ from 'underscore';
+import * as parseIsoDuration from 'parse-iso-duration';
+import * as entities from 'entities';
+import * as cache from './cache.js';
+import * as htmlUtils from './html-utils.js';
+import * as sysUtils from '../logging.js';
 
 var http2Agent = new Http2Agent(/* options */);
 
 if (!global.CONFIG) {
-    global.CONFIG = require('../config');
+    // TODO: review
+     var config = await import('../config.js');
+     global.CONFIG = config.default;
 }
 
 function prepareEncodedUri(request_options, attr) {
@@ -49,7 +49,7 @@ function maxTTL(ttl) {
     return ttl;
 }
 
-var getMaxCacheTTLOverride = exports.getMaxCacheTTLOverride = function(url, options) {
+export function getMaxCacheTTLOverride(url, options) {
     var proxy = null;
     if (CONFIG.PROXY || (options && options.proxy)) {
         proxy = (options && options.proxy) || CONFIG.PROXY.find(p => {
@@ -59,7 +59,7 @@ var getMaxCacheTTLOverride = exports.getMaxCacheTTLOverride = function(url, opti
     return proxy;
 };
 
-var getMaxCacheTTL = exports.getMaxCacheTTL = function(url, options, default_min_ttl) {
+export function getMaxCacheTTL(url, options, default_min_ttl) {
     var proxy = getMaxCacheTTLOverride(url, options);
     var result = Math.max(fixTTL(options && options.cache_ttl), fixTTL(proxy && proxy.cache_ttl), fixTTL(default_min_ttl));
     result = maxTTL(result);
@@ -74,7 +74,7 @@ var getMaxCacheTTL = exports.getMaxCacheTTL = function(url, options, default_min
     return result;
 };
 
-var prepareRequestOptions = exports.prepareRequestOptions = function(request_options, options) {
+export function prepareRequestOptions(request_options, options) {
 
     var url = request_options.uri || request_options.url;
 
@@ -137,7 +137,7 @@ var prepareRequestOptions = exports.prepareRequestOptions = function(request_opt
     return request_options;
 };
 
-var getUrlFunctional = exports.getUrlFunctional = function(url, options, callbacks) {
+export function getUrlFunctional(url, options, callbacks) {
 
     getUrl(url, options)
         .on('error', function(error, opts) {
@@ -176,7 +176,7 @@ var getUrlFunctional = exports.getUrlFunctional = function(url, options, callbac
  * @param {Function} [callback] The completion callback function or events.EventEmitter object
  * @returns {events.EventEmitter} The emitter object which emit error or response event
  */
-var getUrl = exports.getUrl = function(url, options) {
+ export function getUrl(url, options) {
 
     const controller = new AbortController();
 
@@ -286,7 +286,7 @@ var getUrl = exports.getUrl = function(url, options) {
     return req;
 };
 
-var getHeadFunctional = exports.getHeadFunctional = function(url, options, callbacks) {
+export function getHeadFunctional(url, options, callbacks) {
 
     getHead(url, options)
         .on('error', function(error) {
@@ -368,7 +368,7 @@ var getHead = function(url, options) {
     return req;
 };
 
-exports.getCharset = function(string, doNotParse) {
+export function getCharset(string, doNotParse) {
     var charset;
 
     if (doNotParse) {
@@ -381,7 +381,7 @@ exports.getCharset = function(string, doNotParse) {
     return charset;
 };
 
-exports.encodeText = function(charset, text) { 
+export function encodeText(charset, text) { 
     try {
         var charset = charset || 'UTF-8';
 
@@ -407,7 +407,7 @@ exports.encodeText = function(charset, text) {
     }
 };
 
-exports.parseJSONSource = function(text, decode) {
+export function parseJSONSource(text, decode) {
 
     try {
         return JSON.parse(decode ? entities.decodeHTML(decode(text)) : entities.decodeHTML(text));
@@ -491,7 +491,7 @@ exports.parseJSONSource = function(text, decode) {
  *
  *  error == 404 if not found.
  * */
-exports.getImageMetadata = function(uri, options, callback){
+ export function getImageMetadata(uri, options, callback){
 
     if (typeof options === 'function') {
         callback = options;
@@ -646,7 +646,7 @@ exports.getImageMetadata = function(uri, options, callback){
     }, callback);
 };
 
-exports.getUriStatus = function(uri, options, callback) {
+export function getUriStatus(uri, options, callback) {
 
     if (typeof options === 'function') {
         callback = options;
@@ -689,7 +689,7 @@ exports.getUriStatus = function(uri, options, callback) {
             time = createTimer();
         }
 
-        getUriStatus(uri, options, finish);
+        getUriStatusPrivate(uri, options, finish);
 
     }, {
         // Ignore proxy.cache_ttl, if options.cache_ttl === 0 - do not read from cache.
@@ -700,7 +700,7 @@ exports.getUriStatus = function(uri, options, callback) {
     }, callback);
 };
 
-exports.getContentType = function(uriForCache, uriOriginal, options, cb) {
+export function getContentType(uriForCache, uriOriginal, options, cb) {
 
     cache.withCache("content-type:" + uriForCache, function(cb) {
 
@@ -814,7 +814,7 @@ exports.getContentType = function(uriForCache, uriOriginal, options, cb) {
     }, cb);
 };
 
-exports.unifyDuration = function(duration) {
+export function unifyDuration(duration) {
     if (duration && typeof duration === 'string') {
 
         if (duration.match(/^\d+$/)) {
@@ -837,7 +837,7 @@ var NOW = new Date().getTime();
 
 var minDate = new Date(1990, 1);
 
-exports.unifyDate = function(date) {
+export function unifyDate(date) {
 
     if (_.isArray(date)) {
         date = date[0];
@@ -895,7 +895,7 @@ exports.unifyDate = function(date) {
 };
 
 
-var lowerCaseKeys = exports.lowerCaseKeys = function(obj) {
+export function lowerCaseKeys(obj) {
     for (var k in obj) {
         var lowerCaseKey = k.toLowerCase();
         if (lowerCaseKey != k) {
@@ -909,7 +909,7 @@ var lowerCaseKeys = exports.lowerCaseKeys = function(obj) {
     }
 };
 
-exports.sendLogToWhitelist = function(uri, context) {
+export function sendLogToWhitelist(uri, context) {
 
     const {meta, oembed, oembedLinks, whitelistRecord} = context;
 
@@ -982,7 +982,7 @@ exports.sendLogToWhitelist = function(uri, context) {
     }
 };
 
-exports.filterLinks = function(data, options) {
+export function filterLinks(data, options) {
 
     var links = data.links;
 
@@ -1066,7 +1066,7 @@ function iterateLinks(links, func) {
     }
 }
 
-exports.generateLinksHtml = function(data, options) {
+export function generateLinksHtml(data, options) {
 
     // Links may be grouped.
 
@@ -1133,7 +1133,7 @@ exports.generateLinksHtml = function(data, options) {
 // Private
 //====================================================================================
 
-var getUriStatus = function(uri, options, cb) {
+var getUriStatusPrivate = function(uri, options, cb) {
 
     try {
         var r = request(prepareRequestOptions({
@@ -1167,7 +1167,7 @@ var getUriStatus = function(uri, options, cb) {
     }
 };
 
-var createTimer = exports.createTimer = function() {
+export function createTimer() {
 
     var timer = new Date().getTime();
 
@@ -1287,7 +1287,7 @@ function getWhitelistLogData(meta, oembed) {
     return hasTrue && result;
 }
 
-exports.getIframelyErrorShortCode = function(error) {
+export function getIframelyErrorShortCode(error) {
 
     if (error.responseCode) {
         // 'http error'
@@ -1302,7 +1302,7 @@ exports.getIframelyErrorShortCode = function(error) {
     return error.code;
 };
 
-var abortRequest = exports.abortRequest = function(request) {
+export function abortRequest(request) {
     if (request && request.response && request.response.h2) {
         // Abort http2 with special method.
         request.response.abort();
@@ -1316,7 +1316,7 @@ var abortRequest = exports.abortRequest = function(request) {
  * Generate slug for provider based on domain name of URL provided
  * @param url Request uri string
  */
-exports.getProviderName = function(url) {
+export function getProviderName(url) {
 
     try {
         var domain = url.match(/^https?:\/\/([^\/\?#]+)/)[1];
diff --git a/lib/whitelist.js b/lib/whitelist.js
index 15b356b3d..4783f3d60 100644
--- a/lib/whitelist.js
+++ b/lib/whitelist.js
@@ -1,13 +1,17 @@
-(function(whitelist) {
+    import * as chokidar from 'chokidar';
+    import * as fs from 'fs';
+    import * as path from 'path';
+    import * as crypto from 'crypto';
+    import * as _ from 'underscore';
+    import * as utils from './utils.js';
+    import log from '../logging.js';
+    import request from 'request';
 
-    var chokidar = require('chokidar'),
-        fs = require('fs'),
-        path = require('path'),
-        crypto = require('crypto'),
-        _ = require('underscore'),
-        request = require('request'),
-        utils = require('./utils'),
-        logging = require('../logging');
+    import { fileURLToPath } from 'url';
+    import { dirname } from 'path';
+
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
 
     var whitelistObject = {domains: {}};
     var whitelistLastModified;
@@ -93,7 +97,7 @@
         return hash(JSON.stringify(data));
     }
 
-    whitelist.findRawWhitelistRecordFor = function(uri) {
+    export function findRawWhitelistRecordFor(uri) {
 
         if (!whitelistObject || !whitelistObject.domains) {
             return null;
@@ -110,7 +114,7 @@
         return record;
     };
 
-    whitelist.findWhitelistRecordFor = function(uri, options) {
+    export function findWhitelistRecordFor(uri, options) {
 
         if (!whitelistObject) {
             return null;
@@ -165,7 +169,7 @@
         return record;
     };
 
-    whitelist.getWhitelistObject = function() {
+    export function getWhitelistObject() {
         return whitelistObject;
     };
 
@@ -231,7 +235,7 @@
 
         addWildcard();
 
-        logging.log('Whitelist activated. Domains, including blacklisted:', _.keys(data.domains).length);
+        log('Whitelist activated. Domains, including blacklisted:', _.keys(data.domains).length);
     }
 
     function readWhitelist(filename) {
@@ -333,7 +337,7 @@
 
         if (!currentWhitelistFilename && CONFIG.WHITELIST_URL && CONFIG.WHITELIST_URL_RELOAD_PERIOD) {
 
-            logging.log("Loading whitelist from " + CONFIG.WHITELIST_URL);
+            log("Loading whitelist from " + CONFIG.WHITELIST_URL);
 
             var options = {
                 uri: CONFIG.WHITELIST_URL,
@@ -360,7 +364,7 @@
                 } else if (r.statusCode === 500) {
                     console.error('Error loading whitelist from ' + CONFIG.WHITELIST_URL + ' : ' + newWhitelist);
                 } else if (r.statusCode === 304) {
-                    logging.log('Whitelist respond: 304 (not modified)');
+                    log('Whitelist respond: 304 (not modified)');
                 } else if (!newWhitelist || typeof newWhitelist === 'string') {
                     console.error('Error loading whitelist from ' + CONFIG.WHITELIST_URL + ' : incorrect data: ' + newWhitelist);
                 } else {
@@ -373,6 +377,4 @@
                 setTimeout(loadWhitelistUrl, CONFIG.WHITELIST_URL_RELOAD_PERIOD);
             });
         }
-    }
-
-})(exports);
+    }
\ No newline at end of file
diff --git a/logging.js b/logging.js
index 3b0947d26..74d9e68d2 100644
--- a/logging.js
+++ b/logging.js
@@ -1,6 +1,6 @@
-var moment = require('moment');
+import moment from 'moment';
 
-exports.log = function() {
+export default function log() {
     var args = Array.prototype.slice.apply(arguments);
 
     // Add ip if request provided.
diff --git a/modules/api/utils.js b/modules/api/utils.js
index e0993d69b..a2f2d23c2 100644
--- a/modules/api/utils.js
+++ b/modules/api/utils.js
@@ -1,6 +1,6 @@
 var _RE = /^_.+/;
 
-exports.getProviderOptionsQuery = function(query) {
+export function getProviderOptionsQuery(query) {
     var providerOptionsQuery = {};
 
     for(var key in query) {
@@ -28,7 +28,7 @@ function normalizeValue(value) {
     return value;
 }
 
-exports.getProviderOptionsFromQuery = function(query) {
+export function getProviderOptionsFromQuery(query) {
     /*
     Convert '_option=value' to 
     providerOptions = {
diff --git a/modules/api/views.js b/modules/api/views.js
index 181296716..eabb5f0cf 100644
--- a/modules/api/views.js
+++ b/modules/api/views.js
@@ -1,15 +1,15 @@
-var iframelyCore = require('../../lib/core');
-var utils = require('../../utils');
-var _ = require('underscore');
-var async = require('async');
-var cache = require('../../lib/cache');
-var iframelyUtils = require('../../lib/utils');
-var oembedUtils = require('../../lib/oembed');
-var whitelist = require('../../lib/whitelist');
-var pluginLoader = require('../../lib/loader/pluginLoader');
-var jsonxml = require('jsontoxml');
-var url = require('url');
-var apiUtils = require('./utils');
+import * as iframelyCore from '../../lib/core.js';
+import * as utils from '../../utils.js';
+import * as _ from 'underscore';
+import * as async from 'async';
+import * as cache from '../../lib/cache.js';
+import * as iframelyUtils from '../../lib/utils.js';
+import * as oembedUtils from '../../lib/oembed.js';
+import * as whitelist from '../../lib/whitelist.js';
+import * as pluginLoader from '../../lib/loader/pluginLoader.js';
+import * as jsonxml from 'jsontoxml';
+import * as url from 'url';
+import * as apiUtils from './utils.js';
 
 var getProviderOptionsQuery = apiUtils.getProviderOptionsQuery;
 var getProviderOptionsFromQuery = apiUtils.getProviderOptionsFromQuery;
@@ -35,7 +35,9 @@ function prepareUri(uri) {
 
 var log = utils.log;
 
-var version = require('../../package.json').version;
+import { readFile } from 'fs/promises';
+const json = JSON.parse(await readFile(new URL('../../package.json', import.meta.url)));
+var version = json.version;
 
 function getRenderLinkCacheKey(uri, req) {
     var query = getProviderOptionsQuery(req.query);
@@ -106,7 +108,7 @@ function processInitialErrors(uri, next) {
     }
 }
 
-module.exports = function(app) {
+export default function(app) {
 
     app.get('/iframely', function(req, res, next) {
 
diff --git a/modules/debug/views.js b/modules/debug/views.js
index 5b9314aa6..9aae734da 100644
--- a/modules/debug/views.js
+++ b/modules/debug/views.js
@@ -1,6 +1,6 @@
-var getProviderOptionsQuery = require('../api/utils').getProviderOptionsQuery;
+import { getProviderOptionsQuery } from '../api/utils.js';
 
-module.exports = function(app) {
+export default function(app) {
 
     app.get('/debug', function(req, res, next) {
 
diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js
index 890e2cf32..0b70bcf27 100644
--- a/modules/tests-ui/models.js
+++ b/modules/tests-ui/models.js
@@ -1,16 +1,13 @@
-(function() {
-
     if (!CONFIG.tests) {
         return;
     }
-
-    var moment = require('moment');
+    import * as moment from 'moment';
 
     var mongoose, db;
 
     // DB connect.
     try {
-        mongoose = require('mongoose');
+        mongoose = await import('mongoose');
         mongoose.set('useUnifiedTopology', true);
         mongoose.set('useCreateIndex', true);
         mongoose.set('useNewUrlParser', true);
@@ -173,5 +170,3 @@
     exports.PageTestLog = db.model('PageTestLog', PageTestLogSchema);
     exports.TestUrlsSet = db.model('TestUrlsSet', TestUrlsSetSchema);
     exports.TestingProgress = db.model('TestingProgress', TestingProgressSchema);
-
-})();
\ No newline at end of file
diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js
index 8fb7a9fe6..dbcd8b7e7 100644
--- a/modules/tests-ui/tester.js
+++ b/modules/tests-ui/tester.js
@@ -1,4 +1,7 @@
-global.CONFIG = require('../../config');
+import config from '../../config';
+global.CONFIG = config;
+
+global.CONFIG = config.default;
 
 if (!CONFIG.tests) {
     console.error('Tests not started: CONFIG.tests not configured.');
@@ -8,16 +11,13 @@ if (!CONFIG.tests) {
 
 process.title = "iframely-tests";
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-
-var async = require('async');
-var _ = require('underscore');
-
-var models = require('./models');
-var utils = require('./utils');
-
-var iframely = require('../../lib/core').run;
-var whitelist = require('../../lib/whitelist');
-var pluginLoader = require('../../lib/loader/pluginLoader');
+import * as async from 'async';
+import * as _ from 'underscore';
+import * as models from './models.js';
+import * as utils from './utils.js';
+import { run as iframely } from '../../lib/core';
+import * as whitelist from '../../lib/whitelist.js';
+import * as pluginLoader from '../../lib/loader/pluginLoader.js';
 var plugins = pluginLoader._plugins;
 
 var testOnePlugin = false;
diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
index 388040982..583fae091 100644
--- a/modules/tests-ui/utils.js
+++ b/modules/tests-ui/utils.js
@@ -1,20 +1,16 @@
-var _ = require('underscore');
-var FeedParser = require('feedparser');
-var request = require('request');
-var async = require('async');
-var url = require('url');
-
-var models = require('./models');
+import * as _ from 'underscore';
+import * as FeedParser from 'feedparser';
+import * as request from 'request';
+import * as async from 'async';
+import * as url from 'url';
+import * as models from './models.js';
 var PageTestLog = models.PageTestLog;
 var TestUrlsSet = models.TestUrlsSet;
 var PluginTest = models.PluginTest;
-
-var findWhitelistRecordFor = require('../../lib/whitelist').findWhitelistRecordFor;
-
-var iframelyGetPluginData = require('../../lib/core').getPluginData;
-
-var pluginLoader = require('../../lib/loader/pluginLoader');
-var pluginUtils = require('../../lib/loader/utils');
+import { findWhitelistRecordFor as findWhitelistRecordFor } from '../../lib/whitelist';
+import { getPluginData as iframelyGetPluginData } from '../../lib/core';
+import * as pluginLoader from '../../lib/loader/pluginLoader.js';
+import * as pluginUtils from '../../lib/loader/utils.js';
 var plugins = pluginLoader._plugins,
     pluginsList = pluginLoader._pluginsList,
     DEFAULT_PARAMS = [].concat(pluginUtils.DEFAULT_PARAMS, pluginUtils.POST_PLUGIN_DEFAULT_PARAMS),
diff --git a/modules/tests-ui/views.js b/modules/tests-ui/views.js
index 8f4702826..6b01eedf2 100644
--- a/modules/tests-ui/views.js
+++ b/modules/tests-ui/views.js
@@ -1,22 +1,14 @@
-(function() {
-
-    if (!CONFIG.tests) {
-        module.exports = function(){};
-        return;
-    }
-
-    var async = require('async');
-    var moment = require('moment');
-    var _ = require('underscore');
-    var exec = require('child_process').exec;
-
-    var models = require('./models');
-    var utils = require('./utils');
+    import * as async from 'async';
+    import * as moment from 'moment';
+    import * as _ from 'underscore';
+    import { exec as exec } from 'child_process';
+    import * as models from './models.js';
+    import * as utils from './utils.js';
 
     var PluginTest = models.PluginTest;
     var TestingProgress = models.TestingProgress;
 
-    module.exports = function(app){
+    export default function(app){
 
         app.get('/tests/run/:plugin', function(req, res, next) {
 
diff --git a/package.json b/package.json
index b8084673f..4127c57fa 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,7 @@
 {
     "name": "iframely",
     "version": "1.6.1",
+    "type": "module",
     "description": "oEmbed/2 gateway endpoint. Get embed data for various http links through one self-hosted API",
     "keywords": [
         "oembed",
@@ -26,6 +27,7 @@
         "ejs": "^3.1.6",
         "entities": "1.1.1",
         "express": "^4.16.3",
+        "globby": "^12.0.2",
         "graceful-cluster": "0.0.3",
         "htmlparser2": "3.9.2",
         "http-parser-js": "itteco/http-parser-js#magicode-fix-22",
diff --git a/server.js b/server.js
index 1ba753503..b82637a72 100644
--- a/server.js
+++ b/server.js
@@ -1,5 +1,5 @@
-var sysUtils = require('./utils');
-var app = require('./app');
+import * as sysUtils from './utils.js';
+import * as app from './app.js';
 
 var server = app.listen(process.env.PORT || CONFIG.port, process.env.HOST || CONFIG.host, function(){
     console.log('\niframely is running on ' + server.address().address + ':' + server.address().port);
diff --git a/utils.js b/utils.js
index 18d8e9f4d..ae83c7fb2 100644
--- a/utils.js
+++ b/utils.js
@@ -1,21 +1,20 @@
-(function() {
+    import config from './config.js';
+    global.CONFIG = config;
 
-    global.CONFIG = require('./config');
+    import * as async from 'async';
+    import * as cache from './lib/cache.js';
+    import * as ejs from 'ejs';
+    import * as fs from 'fs';
+    import * as crypto from 'crypto';
+    import * as _ from 'underscore';
+    import * as urlLib from 'url';
 
-    var async = require('async');
-    var cache = require('./lib/cache');
-    var ejs = require('ejs');
-    var fs = require('fs');
-    var crypto = require('crypto');
-    var _ = require('underscore');
-    var urlLib = require('url');
+    import log from './logging.js';
+    export {log};
+    import * as whitelist from './lib/whitelist.js';
+    import * as pluginLoader from './lib/loader/pluginLoader.js';
 
-    var log = exports.log = require('./logging').log;
-
-    var whitelist = require('./lib/whitelist');
-    var pluginLoader = require('./lib/loader/pluginLoader');
-
-    function NotFound(message, messages) {
+    export function NotFound(message, messages) {
 
         if (typeof message === 'object') {
             this.meta = message;
@@ -33,9 +32,7 @@
 
     NotFound.prototype.__proto__ = Error.prototype;
 
-    exports.NotFound = NotFound;
-
-    function HttpError(code, message, messages) {
+    export function HttpError(code, message, messages) {
 
         Error.call(this); //super constructor
         Error.captureStackTrace(this, this.constructor); //super helper method to include stack trace in error object
@@ -49,9 +46,9 @@
 
     HttpError.prototype.__proto__ = Error.prototype;
 
-    exports.HttpError = HttpError;
-
-    var version = require('./package.json').version;
+    import { readFile } from 'fs/promises';
+    const json = JSON.parse(await readFile(new URL('./package.json', import.meta.url)));
+    var version = json.version;
 
     var etag = function(value) {
         return '"' + crypto.createHash('md5').update(value).digest("hex") + '"';
@@ -158,7 +155,7 @@
         cache.set('urlcache:' + version + (linkValidationKey || '') + ':' + url, data, {ttl: ttl});
     }
 
-    exports.cacheMiddleware = function(req, res, next) {
+    export function cacheMiddleware(req, res, next) {
 
         async.waterfall([
 
@@ -334,5 +331,3 @@
             next();
         });
     };
-
-})();
diff --git a/yarn.lock b/yarn.lock
index 4ddf3e494..8ba060599 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,6 +2,27 @@
 # yarn lockfile v1
 
 
+"@nodelib/fs.scandir@2.1.5":
+  version "2.1.5"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
+  integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==
+  dependencies:
+    "@nodelib/fs.stat" "2.0.5"
+    run-parallel "^1.1.9"
+
+"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2":
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b"
+  integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==
+
+"@nodelib/fs.walk@^1.2.3":
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a"
+  integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==
+  dependencies:
+    "@nodelib/fs.scandir" "2.1.5"
+    fastq "^1.6.0"
+
 "@types/bson@*":
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.0.3.tgz#30889d2ffde6262abbe38659364c631454999fbf"
@@ -69,7 +90,7 @@ ansi-styles@^3.2.1:
   dependencies:
     color-convert "^1.9.0"
 
-anymatch@~3.1.1:
+anymatch@~3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716"
   integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==
@@ -100,6 +121,11 @@ array-indexofobject@~0.0.1:
   resolved "https://registry.yarnpkg.com/array-indexofobject/-/array-indexofobject-0.0.1.tgz#aaa128e62c9b3c358094568c219ff64fe489d42a"
   integrity sha1-qqEo5iybPDWAlFaMIZ/2T+SJ1Co=
 
+array-union@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/array-union/-/array-union-3.0.1.tgz#da52630d327f8b88cfbfb57728e2af5cd9b6b975"
+  integrity sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw==
+
 asn1@~0.2.3:
   version "0.2.4"
   resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
@@ -217,7 +243,7 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
-braces@~3.0.2:
+braces@^3.0.1, braces@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
   integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==
@@ -298,19 +324,19 @@ cheerio@0.22.0:
     lodash.some "^4.4.0"
 
 chokidar@^3.3.1:
-  version "3.4.3"
-  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.4.3.tgz#c1df38231448e45ca4ac588e6c79573ba6a57d5b"
-  integrity sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ==
+  version "3.5.2"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.2.tgz#dba3976fcadb016f66fd365021d91600d01c1e75"
+  integrity sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ==
   dependencies:
-    anymatch "~3.1.1"
+    anymatch "~3.1.2"
     braces "~3.0.2"
-    glob-parent "~5.1.0"
+    glob-parent "~5.1.2"
     is-binary-path "~2.1.0"
     is-glob "~4.0.1"
     normalize-path "~3.0.0"
-    readdirp "~3.5.0"
+    readdirp "~3.6.0"
   optionalDependencies:
-    fsevents "~2.1.2"
+    fsevents "~2.3.2"
 
 chownr@^1.1.1:
   version "1.1.4"
@@ -520,6 +546,13 @@ diff@3.5.0:
   resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
   integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==
 
+dir-glob@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
+  integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==
+  dependencies:
+    path-type "^4.0.0"
+
 dom-serializer@0, dom-serializer@~0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
@@ -690,11 +723,29 @@ fast-deep-equal@^3.1.1:
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
   integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==
 
+fast-glob@^3.2.7:
+  version "3.2.7"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1"
+  integrity sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q==
+  dependencies:
+    "@nodelib/fs.stat" "^2.0.2"
+    "@nodelib/fs.walk" "^1.2.3"
+    glob-parent "^5.1.2"
+    merge2 "^1.3.0"
+    micromatch "^4.0.4"
+
 fast-json-stable-stringify@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
   integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==
 
+fastq@^1.6.0:
+  version "1.13.0"
+  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c"
+  integrity sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==
+  dependencies:
+    reusify "^1.0.4"
+
 feedparser@2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/feedparser/-/feedparser-2.2.0.tgz#90279e46af711649a2135307085c9377865adcf3"
@@ -782,10 +833,10 @@ fs.realpath@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
 
-fsevents@~2.1.2:
-  version "2.1.3"
-  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.1.3.tgz#fb738703ae8d2f9fe900c33836ddebee8b97f23e"
-  integrity sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==
+fsevents@~2.3.2:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
 
 gauge@~2.7.3:
   version "2.7.4"
@@ -813,7 +864,7 @@ github-from-package@0.0.0:
   resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce"
   integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=
 
-glob-parent@~5.1.0:
+glob-parent@^5.1.2, glob-parent@~5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
   integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
@@ -844,6 +895,18 @@ glob@~7.1.3:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
+globby@^12.0.2:
+  version "12.0.2"
+  resolved "https://registry.yarnpkg.com/globby/-/globby-12.0.2.tgz#53788b2adf235602ed4cabfea5c70a1139e1ab11"
+  integrity sha512-lAsmb/5Lww4r7MM9nCCliDZVIKbZTavrsunAsHLr9oHthrZP1qi7/gAnHOsUs9bLvEt2vKVJhHmxuL7QbDuPdQ==
+  dependencies:
+    array-union "^3.0.1"
+    dir-glob "^3.0.1"
+    fast-glob "^3.2.7"
+    ignore "^5.1.8"
+    merge2 "^1.4.1"
+    slash "^4.0.0"
+
 graceful-cluster@0.0.3:
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/graceful-cluster/-/graceful-cluster-0.0.3.tgz#2c3f57da3610afb28c1bbc8afaa5030dcf506bdc"
@@ -974,6 +1037,11 @@ ieee754@^1.1.13:
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
   integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
 
+ignore@^5.1.8:
+  version "5.1.8"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
+  integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==
+
 iltorb@^2.4.3:
   version "2.4.5"
   resolved "https://registry.yarnpkg.com/iltorb/-/iltorb-2.4.5.tgz#d64434b527099125c6839ed48b666247a172ef87"
@@ -1033,9 +1101,9 @@ is-fullwidth-code-point@^1.0.0:
     number-is-nan "^1.0.0"
 
 is-glob@^4.0.1, is-glob@~4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc"
-  integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==
+  version "4.0.3"
+  resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
+  integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==
   dependencies:
     is-extglob "^2.1.1"
 
@@ -1239,11 +1307,24 @@ merge-descriptors@1.0.1:
   resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61"
   integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=
 
+merge2@^1.3.0, merge2@^1.4.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
+  integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
+
 methods@^1.1.1, methods@^1.1.2, methods@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=
 
+micromatch@^4.0.4:
+  version "4.0.4"
+  resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9"
+  integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==
+  dependencies:
+    braces "^3.0.1"
+    picomatch "^2.2.3"
+
 mime-db@1.44.0:
   version "1.44.0"
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92"
@@ -1575,15 +1656,20 @@ path-to-regexp@0.1.7:
   resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c"
   integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=
 
+path-type@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
+  integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
+
 performance-now@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
   integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
 
-picomatch@^2.0.4, picomatch@^2.2.1:
-  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.3.tgz#465547f359ccc206d3c48e46a1bcb89bf7ee619d"
-  integrity sha512-KpELjfwcCDUb9PeigTs2mBJzXUPzAuP2oPcA989He8Rte0+YUAjw1JVedDhuTKPkHjSYzMN3npC9luThGYEKdg==
+picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
+  integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
 
 prebuild-install@^5.3.3:
   version "5.3.6"
@@ -1663,6 +1749,11 @@ qs@~6.5.2:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
   integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
 
+queue-microtask@^1.2.2:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"
+  integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
+
 random-bytes@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b"
@@ -1748,10 +1839,10 @@ readable-stream@~2.1.5:
     string_decoder "~0.10.x"
     util-deprecate "~1.0.1"
 
-readdirp@~3.5.0:
-  version "3.5.0"
-  resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.5.0.tgz#9ba74c019b15d365278d2e91bb8c48d7b4d42c9e"
-  integrity sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==
+readdirp@~3.6.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7"
+  integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==
   dependencies:
     picomatch "^2.2.1"
 
@@ -1851,6 +1942,18 @@ retry@0.6.0:
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.6.0.tgz#1c010713279a6fd1e8def28af0c3ff1871caa537"
   integrity sha1-HAEHEyeab9Ho3vKK8MP/GHHKpTc=
 
+reusify@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
+  integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
+
+run-parallel@^1.1.9:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee"
+  integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==
+  dependencies:
+    queue-microtask "^1.2.2"
+
 safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
@@ -1985,6 +2088,11 @@ simple-lru-cache@0.0.x:
   resolved "https://registry.yarnpkg.com/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz#d59cc3a193c1a5d0320f84ee732f6e4713e511dd"
   integrity sha1-1ZzDoZPBpdAyD4Tucy9uRxPlEd0=
 
+slash@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/slash/-/slash-4.0.0.tgz#2422372176c4c6c5addb5e2ada885af984b396a7"
+  integrity sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==
+
 sliced@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/sliced/-/sliced-1.0.1.tgz#0b3a662b5d04c3177b1926bea82b03f837a2ef41"

From 0ee79cff8eab84c1a21cc8420a175a4cd2783381 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 14 Oct 2021 15:59:37 +0300
Subject: [PATCH 007/102] tests: ES6 updates

---
 test/core-plugins.js                             | 11 ++++++-----
 test/custom_plugins.js                           | 11 +++++------
 test/e2e.js                                      | 11 +++++------
 test/fixtures/custom-plugins/domains/test.com.js |  4 +---
 test/fixtures/custom-plugins/meta/html-title.js  |  2 +-
 5 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/test/core-plugins.js b/test/core-plugins.js
index 4191ad864..86195b445 100644
--- a/test/core-plugins.js
+++ b/test/core-plugins.js
@@ -1,10 +1,11 @@
-var assert = require('chai').assert
+import { assert as assert } from 'chai'
 
-global.CONFIG = require('../config');
+import config from '../config';
+global.CONFIG = config;
 
-var iframely = require('../lib/core').getPluginData;
-var findWhitelistRecordFor = require('../lib/whitelist').findWhitelistRecordFor;
-var utils = require('../lib/utils');
+import { getPluginData as iframely } from '../lib/core';
+import { findWhitelistRecordFor as findWhitelistRecordFor } from '../lib/whitelist';
+import * as utils from '../lib/utils.js';
 
 function assertOembed(oembed) {
     assert.isObject(oembed);
diff --git a/test/custom_plugins.js b/test/custom_plugins.js
index 72aef2a18..b8b280f9f 100644
--- a/test/custom_plugins.js
+++ b/test/custom_plugins.js
@@ -1,9 +1,8 @@
 'use strict';
-
-var request = require('supertest');
-var ServerMock = require('mock-http-server');
-var chai = require('chai');
-var async = require('async');
+import * as request from 'supertest';
+import * as ServerMock from 'mock-http-server';
+import * as chai from 'chai';
+import * as async from 'async';
 
 var server;
 
@@ -15,7 +14,7 @@ function invalidateRequireCache () {
 
 var startWithENV = function (env, cb) {
   process.env.NODE_ENV = env;
-  var app = require('../app');
+import * as app from '../app.js';
   server = app.listen(process.env.PORT, cb);
 };
 
diff --git a/test/e2e.js b/test/e2e.js
index 63d6e3a7d..8173f50d1 100644
--- a/test/e2e.js
+++ b/test/e2e.js
@@ -1,9 +1,8 @@
 'use strict';
-
-var request = require('supertest');
-var ServerMock = require('mock-http-server');
-var chai = require('chai');
-var async = require('async');
+import * as request from 'supertest';
+import * as ServerMock from 'mock-http-server';
+import * as chai from 'chai';
+import * as async from 'async';
 
 describe('meta endpoint', function() {
 
@@ -16,7 +15,7 @@ describe('meta endpoint', function() {
   var server;
 
   beforeEach(function(done) {
-    var app = require('../app');
+import * as app from '../app.js';
     server = app.listen(process.env.PORT, function() {
       targetMockedServer.start(done);
     });
diff --git a/test/fixtures/custom-plugins/domains/test.com.js b/test/fixtures/custom-plugins/domains/test.com.js
index 5772a1adf..0bdebfebc 100644
--- a/test/fixtures/custom-plugins/domains/test.com.js
+++ b/test/fixtures/custom-plugins/domains/test.com.js
@@ -1,6 +1,4 @@
-var utils = require('../../../../utils');
-
-module.exports = {
+export default {
 
     re: /testok/i,
 
diff --git a/test/fixtures/custom-plugins/meta/html-title.js b/test/fixtures/custom-plugins/meta/html-title.js
index 11379bedc..d0b682d25 100644
--- a/test/fixtures/custom-plugins/meta/html-title.js
+++ b/test/fixtures/custom-plugins/meta/html-title.js
@@ -1,4 +1,4 @@
-module.exports = {
+export default {
 
     getMeta: function(meta) {
         return {

From e5a824d6dca51d1abaf000d0dc5da4ceb6a2432b Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 14 Oct 2021 17:34:36 +0300
Subject: [PATCH 008/102] remove all old exports

---
 .../templates/config.local.js.j2              |  8 ++-----
 app.js                                        |  7 ++----
 config.js                                     |  4 ++--
 config.local.js.SAMPLE                        |  8 ++-----
 config.test.js                                | 10 ++------
 config.test_with_custom_plugins.js            |  9 ++-----
 lib/cache-engines/memcached.js                | 20 ++++++++--------
 lib/cache-engines/no-cache.js                 | 10 +++-----
 lib/cache-engines/redis.js                    | 20 +++++++---------
 lib/loader/pluginLoader.js                    |  4 ----
 .../CollectingHandlerForMutliTarget.js        |  6 ++---
 lib/plugins/system/meta/HTMLMetaHandler.js    |  8 +++----
 lib/plugins/system/meta/ld-json.js            |  4 ++--
 lib/plugins/system/meta/utils.js              |  4 ++--
 lib/plugins/system/oembed/oembedUtils.js      |  6 ++---
 modules/tests-ui/models.js                    | 11 ++++-----
 modules/tests-ui/utils.js                     | 24 +++++++++----------
 plugins/custom/noindex/utils.js               |  4 ++--
 plugins/meta/description-from-p-tag.js        |  2 +-
 server.js                                     |  4 +---
 20 files changed, 64 insertions(+), 109 deletions(-)

diff --git a/ansible-docker-iframely/templates/config.local.js.j2 b/ansible-docker-iframely/templates/config.local.js.j2
index bde5aaa8c..bbf786599 100644
--- a/ansible-docker-iframely/templates/config.local.js.j2
+++ b/ansible-docker-iframely/templates/config.local.js.j2
@@ -1,5 +1,4 @@
-(function() {
-    var config = {
+    export default {
 
         // Specify a path for custom plugins. Custom plugins will override core plugins.
         // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder',
@@ -259,7 +258,4 @@
               }
         }
         */
-    };
-
-    module.exports = config;
-})();
+    };
\ No newline at end of file
diff --git a/app.js b/app.js
index 77c03a36d..2718ae0ba 100644
--- a/app.js
+++ b/app.js
@@ -8,13 +8,12 @@ if (!CONFIG.baseAppUrl) {
   console.warn('Warning: CONFIG.baseAppUrl not set, default value used');
 }
 
-import * as path from 'path';
 import * as express from 'express';
 import * as jsonxml from 'jsontoxml';
 
 var NotFound = sysUtils.NotFound;
 
-var app = express();
+export const app = express();
 
 app.set('view engine', 'ejs');
 
@@ -174,6 +173,4 @@ app.get('/', function(req, res) {
 });
 
 process.title = "iframely";
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-
-module.exports = app;
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
\ No newline at end of file
diff --git a/config.js b/config.js
index 6da7fe620..83472b6ab 100644
--- a/config.js
+++ b/config.js
@@ -12,7 +12,7 @@
     const json = JSON.parse(await readFile(new URL('./package.json', import.meta.url)));
     var version = json.version;
 
-    var config = {
+    const config = {
 
         baseAppUrl: "",
         port: 8061,
@@ -430,4 +430,4 @@
         config.HTTP2_RETRY_CODES[item] = 1;
     });
 
-    export default config;
+    export default config;
\ No newline at end of file
diff --git a/config.local.js.SAMPLE b/config.local.js.SAMPLE
index 02382972b..88a8cf8e6 100644
--- a/config.local.js.SAMPLE
+++ b/config.local.js.SAMPLE
@@ -1,5 +1,4 @@
-(function() {
-    var config = {
+    export default {
 
         // Specify a path for custom plugins. Custom plugins will override core plugins.
         // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder',
@@ -325,7 +324,4 @@
         // Endpoint for prerender service, if you need it. Used to parse React apps. Very slow.
         // Tested with https://github.com/prerender/prerender
         // PRERENDER_URL: "https://domain/render?url="
-    };
-
-    module.exports = config;
-})();
+    };
\ No newline at end of file
diff --git a/config.test.js b/config.test.js
index 1a4ee3ee1..0461a4ef4 100644
--- a/config.test.js
+++ b/config.test.js
@@ -1,7 +1,4 @@
-(function() {
-
-  var config = {
-
+export default {
     DEBUG: true,
     RICH_LOG_ENABLED: true,
 
@@ -11,7 +8,4 @@
     RESPONSE_TIMEOUT: 1 * 100, //ms
 
     IGNORE_DOMAINS_RE: /blacklisted.*/
-  };
-
-  module.exports = config;
-})();
\ No newline at end of file
+};
\ No newline at end of file
diff --git a/config.test_with_custom_plugins.js b/config.test_with_custom_plugins.js
index 2e36a421a..690852dbe 100644
--- a/config.test_with_custom_plugins.js
+++ b/config.test_with_custom_plugins.js
@@ -1,6 +1,4 @@
-(function() {
-
-  var config = {
+  export default {
 
     CUSTOM_PLUGINS_PATH: __dirname + '/test/fixtures/custom-plugins',
 
@@ -14,7 +12,4 @@
     RESPONSE_TIMEOUT: 1 * 100, //ms
 
     IGNORE_DOMAINS_RE: /blacklisted.*/
-  };
-
-  module.exports = config;
-})();
\ No newline at end of file
+  };
\ No newline at end of file
diff --git a/lib/cache-engines/memcached.js b/lib/cache-engines/memcached.js
index 1868c3f94..3cb04fa1b 100644
--- a/lib/cache-engines/memcached.js
+++ b/lib/cache-engines/memcached.js
@@ -1,7 +1,9 @@
-(function(engine) {
-import * as sysUtils from '../../logging.js';
-import * as crypto from 'crypto';
-import * as Memcached from 'memcached';
+
+    import * as sysUtils from '../../logging.js';
+    import * as crypto from 'crypto';
+    import * as Memcached from 'memcached';
+    import * as _ from 'underscore';
+
     var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options);
 
     var timeout = CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options && CONFIG.MEMCACHED_OPTIONS.options.timeout;
@@ -13,7 +15,7 @@ import * as Memcached from 'memcached';
     }
 
     function _findKeyMeta(k) {
-import * as _ from 'underscore';
+
 
         var sk = safeKey(k);
 
@@ -51,7 +53,7 @@ import * as _ from 'underscore';
         });
     }
 
-    engine.set = function(_key, data, options) {
+    export function set(_key, data, options) {
 
         var key = (!options || !options.raw) ? safeKey(_key) : _key;
 
@@ -75,7 +77,7 @@ import * as _ from 'underscore';
         }
     };
 
-    engine.get = function(_key, cb) {
+    export function get(_key, cb) {
 
         var key = safeKey(_key);
 
@@ -127,8 +129,6 @@ import * as _ from 'underscore';
         });
     };
 
-    engine.getClient = function() {
+    export function getClient() {
         return memcached;
     };
-
-})(exports);
\ No newline at end of file
diff --git a/lib/cache-engines/no-cache.js b/lib/cache-engines/no-cache.js
index 2bd2cc32b..500c6ebb3 100644
--- a/lib/cache-engines/no-cache.js
+++ b/lib/cache-engines/no-cache.js
@@ -1,11 +1,7 @@
-(function(engine) {
-
-    engine.set = function(key, data) {
+    export function set(key, data) {
 
     };
 
-    engine.get = function(key, cb) {
+    export function get(key, cb) {
         cb(null, null);
-    };
-
-})(exports);
\ No newline at end of file
+    };
\ No newline at end of file
diff --git a/lib/cache-engines/redis.js b/lib/cache-engines/redis.js
index 33ffc6c2a..68c42d956 100644
--- a/lib/cache-engines/redis.js
+++ b/lib/cache-engines/redis.js
@@ -1,18 +1,16 @@
-(function(engine) {
-import * as sysUtils from '../../logging.js';
+    import * as sysUtils from '../../logging.js';
 
     var client;
 
     if (CONFIG.REDIS_MODE === 'cluster') {
-import * as RedisClustr from 'redis-clustr';
-
-        client = new RedisClustr(CONFIG.REDIS_CLUSTER_OPTIONS);
+        var pkg = await import('redis-clustr');
+        client = new pkg.RedisClustr(CONFIG.REDIS_CLUSTER_OPTIONS);
     } else {
-import * as redis from 'redis';
-        client = redis.createClient(CONFIG.REDIS_OPTIONS);
+        var pkg = await import('redis');
+        client = pkg.redis.createClient(CONFIG.REDIS_OPTIONS);
     }
 
-    engine.set = function(key, data, options) {
+    export function set(key, data, options) {
         var multi = client.multi();
         multi.set(key, JSON.stringify(data));
         multi.expire(key, options && options.ttl || CONFIG.CACHE_TTL);
@@ -24,7 +22,7 @@ import * as redis from 'redis';
         });
     };
 
-    engine.get = function(key, cb) {
+    export function get(key, cb) {
 
         client.get(key, function(error, data) {
 
@@ -45,6 +43,4 @@ import * as redis from 'redis';
 
             cb(null, parsedData);
         });
-    };
-
-})(exports);
+    };
\ No newline at end of file
diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js
index 8c53d3501..93be9a359 100644
--- a/lib/loader/pluginLoader.js
+++ b/lib/loader/pluginLoader.js
@@ -162,10 +162,6 @@
                 return;
             }
 
-            if (!plugin) {
-                console.error("Error loading plugin", pluginPath);
-            }
-
             if (notPlugin || plugin.notPlugin) {
                 // Skip utils modules.
                 return;
diff --git a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
index 4956f13fa..a9aa4ce54 100644
--- a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
+++ b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
@@ -5,7 +5,7 @@
 *
 * */
 
-function CollectingHandlerForMutliTarget(cbsArray){
+export function CollectingHandlerForMutliTarget(cbsArray){
     this._cbsArray = cbsArray || [];
     this.events = [];
 }
@@ -143,6 +143,4 @@ CollectingHandlerForMutliTarget.prototype._emitEventsFor = function(cbs) {
     }
 };
 
-module.exports = CollectingHandlerForMutliTarget;
-
-module.exports.notPlugin = true;
\ No newline at end of file
+export const notPlugin = true;
\ No newline at end of file
diff --git a/lib/plugins/system/meta/HTMLMetaHandler.js b/lib/plugins/system/meta/HTMLMetaHandler.js
index 7827d5715..590ea2bc9 100644
--- a/lib/plugins/system/meta/HTMLMetaHandler.js
+++ b/lib/plugins/system/meta/HTMLMetaHandler.js
@@ -2,7 +2,7 @@ import { decodeHTML5 } from 'entities';
 import * as _ from 'underscore';
 import * as url from 'url';
 import * as utils from '../../../utils.js';
-import * as ldParser from './ld-json.js';
+import { ldParser } from './ld-json.js';
 
 var getCharset = utils.getCharset;
 var encodeText = utils.encodeText;
@@ -23,7 +23,7 @@ var LINK_REL_ARRAY_VALUES = [
     'alternative'
 ];
 
-function HTMLMetaHandler(uri, contentType, callback) {
+export function HTMLMetaHandler(uri, contentType, callback) {
     this._uri = uri;
     this._charset = getCharset(contentType);
     this._callback = callback;
@@ -582,6 +582,4 @@ function merge(parentObj, props, value) {
     }
 }
 
-module.exports = HTMLMetaHandler;
-
-module.exports.notPlugin = true;
+export const notPlugin = true;
diff --git a/lib/plugins/system/meta/ld-json.js b/lib/plugins/system/meta/ld-json.js
index b5d7abe4a..9984702b7 100644
--- a/lib/plugins/system/meta/ld-json.js
+++ b/lib/plugins/system/meta/ld-json.js
@@ -1,7 +1,7 @@
 import * as sysUtils from '../../../../logging.js';
 import * as utils from '../../../utils.js';
 
-module.exports = function(result, decode, uri) {
+export function ldParser(result, decode, uri) {
 
     var ld = result["ld-json"];
     delete result["ld-json"];
@@ -51,4 +51,4 @@ module.exports = function(result, decode, uri) {
     }    
 };
 
-module.exports.notPlugin = true;
\ No newline at end of file
+export const notPlugin = true;
\ No newline at end of file
diff --git a/lib/plugins/system/meta/utils.js b/lib/plugins/system/meta/utils.js
index 90305ab42..ff5c1fa70 100644
--- a/lib/plugins/system/meta/utils.js
+++ b/lib/plugins/system/meta/utils.js
@@ -1,4 +1,4 @@
-exports.getMetaCacheKey = function(url, whitelistRecord, options) {
+export function getMetaCacheKey(url, whitelistRecord, options) {
 
     var meta_key = 'meta:' + url;
 
@@ -15,4 +15,4 @@ exports.getMetaCacheKey = function(url, whitelistRecord, options) {
     return meta_key;
 };
 
-module.exports.notPlugin = true;
\ No newline at end of file
+export const notPlugin = true;
\ No newline at end of file
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index 6f339a39c..1e87a119c 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -11,7 +11,7 @@ var getCharset = utils.getCharset;
 var encodeText = utils.encodeText;
 var lowerCaseKeys = utils.lowerCaseKeys;
 
-exports.notPlugin = true;
+export const notPlugin = true;
 
 /**
  * @private
@@ -84,7 +84,7 @@ function lookupStaticProviders(uri) {
     return links;
 }
 
-module.exports.findOembedLinks = function(uri, meta) {
+export function findOembedLinks(uri, meta) {
     // Filter oembed from meta.
 
     // allow misspelled discovery links
@@ -126,7 +126,7 @@ module.exports.findOembedLinks = function(uri, meta) {
  * @param {String} uri Full oEmbed endpoint plus URL and any needed format parameter.
  * @param {Function} callback Completion callback function. The callback gets two arguments (error, oembed) where oembed is json parsed oEmbed object.
  * */
-module.exports.getOembed = function(uri, options, callback) {
+ export function getOembed(uri, options, callback) {
 
     if (typeof options === 'function') {
         callback = options;
diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js
index 0b70bcf27..2c6320141 100644
--- a/modules/tests-ui/models.js
+++ b/modules/tests-ui/models.js
@@ -1,6 +1,3 @@
-    if (!CONFIG.tests) {
-        return;
-    }
     import * as moment from 'moment';
 
     var mongoose, db;
@@ -166,7 +163,7 @@
         return moment(this.created_at).format("DD-MM-YY HH:mm");
     };
 
-    exports.PluginTest = db.model('PluginTest', PluginTestSchema);
-    exports.PageTestLog = db.model('PageTestLog', PageTestLogSchema);
-    exports.TestUrlsSet = db.model('TestUrlsSet', TestUrlsSetSchema);
-    exports.TestingProgress = db.model('TestingProgress', TestingProgressSchema);
+    export const PluginTest = db.model('PluginTest', PluginTestSchema);
+    export const PageTestLog = db.model('PageTestLog', PageTestLogSchema);
+    export const TestUrlsSet = db.model('TestUrlsSet', TestUrlsSetSchema);
+    export const TestingProgress = db.model('TestingProgress', TestingProgressSchema);
\ No newline at end of file
diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
index 583fae091..c9445205b 100644
--- a/modules/tests-ui/utils.js
+++ b/modules/tests-ui/utils.js
@@ -3,14 +3,12 @@ import * as FeedParser from 'feedparser';
 import * as request from 'request';
 import * as async from 'async';
 import * as url from 'url';
-import * as models from './models.js';
-var PageTestLog = models.PageTestLog;
-var TestUrlsSet = models.TestUrlsSet;
-var PluginTest = models.PluginTest;
-import { findWhitelistRecordFor as findWhitelistRecordFor } from '../../lib/whitelist';
+import { PageTestLog, TestUrlsSet, PluginTest } from './models.js';
+import { findWhitelistRecordFor } from '../../lib/whitelist';
 import { getPluginData as iframelyGetPluginData } from '../../lib/core';
 import * as pluginLoader from '../../lib/loader/pluginLoader.js';
 import * as pluginUtils from '../../lib/loader/utils.js';
+
 var plugins = pluginLoader._plugins,
     pluginsList = pluginLoader._pluginsList,
     DEFAULT_PARAMS = [].concat(pluginUtils.DEFAULT_PARAMS, pluginUtils.POST_PLUGIN_DEFAULT_PARAMS),
@@ -33,7 +31,7 @@ const COLORS = {
 
 const SLACK_USERNAME = "Testy";
 
-exports.sendQANotification = function(logEntry, data) {
+export function sendQANotification(logEntry, data) {
 
     if (CONFIG.SLACK_WEBHOOK_FOR_QA && CONFIG.SLACK_CHANNEL_FOR_QA) {
 
@@ -89,7 +87,7 @@ exports.sendQANotification = function(logEntry, data) {
 }
 
 function getTestsSummary(cb) {
-    exports.loadPluginTests(function(error, pluginTests) {
+    loadPluginTests(function(error, pluginTests) {
 
         pluginTests.forEach(function(pluginTest) {
 
@@ -125,7 +123,7 @@ function getTestsSummary(cb) {
     });
 }
 
-exports.loadPluginTests = function(cb) {
+export function loadPluginTests(cb) {
 
     var pluginTests;
 
@@ -212,7 +210,7 @@ exports.loadPluginTests = function(cb) {
     });
 }
 
-exports.getPluginUnusedMethods = function(pluginId, debugData) {
+export function getPluginUnusedMethods(pluginId, debugData) {
 
     var usedMethods = getAllUsedMethods(debugData);
     var pluginMethods = findAllPluginMethods(pluginId, plugins);
@@ -224,7 +222,7 @@ exports.getPluginUnusedMethods = function(pluginId, debugData) {
     };
 };
 
-exports.getErrors = function(debugData) {
+export function getErrors(debugData) {
 
     var errors = [];
 
@@ -244,7 +242,7 @@ exports.getErrors = function(debugData) {
 
 var MAX_FEED_URLS = 5;
 
-var fetchFeedUrls = exports.fetchFeedUrls = function(feedUrl, options, cb) {
+export function fetchFeedUrls(feedUrl, options, cb) {
 
     if (typeof options === "function") {
         cb = options;
@@ -296,7 +294,7 @@ var fetchFeedUrls = exports.fetchFeedUrls = function(feedUrl, options, cb) {
         });
 };
 
-exports.fetchUrlsByPageOnFeed = function(pageWithFeed, otpions, cb) {
+export function fetchUrlsByPageOnFeed(pageWithFeed, otpions, cb) {
 
     if (typeof options === "function") {
         cb = options;
@@ -339,7 +337,7 @@ exports.fetchUrlsByPageOnFeed = function(pageWithFeed, otpions, cb) {
     ], cb);
 };
 
-exports.fetchUrlsByPageAndSelector = function(page, selector, options, cb) {
+export function fetchUrlsByPageAndSelector(page, selector, options, cb) {
 
     if (typeof options === "function") {
         cb = options;
diff --git a/plugins/custom/noindex/utils.js b/plugins/custom/noindex/utils.js
index 4f8e0889c..2b653581d 100644
--- a/plugins/custom/noindex/utils.js
+++ b/plugins/custom/noindex/utils.js
@@ -1,8 +1,8 @@
-exports.notPlugin = true;
+export const notPlugin = true;
 
 var NO_INDEX_TAGS = ['noindex'];
 
-exports.checkRobots = function(noindexHeader, cb) {
+export function checkRobots(noindexHeader, cb) {
     if (noindexHeader) {
         var i;
         for(i = 0; i < NO_INDEX_TAGS.length; i++) {
diff --git a/plugins/meta/description-from-p-tag.js b/plugins/meta/description-from-p-tag.js
index 7a4659590..9bd81dfba 100644
--- a/plugins/meta/description-from-p-tag.js
+++ b/plugins/meta/description-from-p-tag.js
@@ -1,4 +1,4 @@
-import { decodeHTML5 as decodeHTML5 } from 'entities';
+import { decodeHTML5 } from 'entities';
 
 export default {
 
diff --git a/server.js b/server.js
index b82637a72..cf5b8edd2 100644
--- a/server.js
+++ b/server.js
@@ -22,6 +22,4 @@ if (!CONFIG.DEBUG) {
         log: sysUtils.log,
         shutdownTimeout: CONFIG.SHUTDOWN_TIMEOUT
     });
-}
-
-module.exports = server;
+}
\ No newline at end of file

From ff33a7c45038d6b4e343effa3fd23323a3ca5e59 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 14 Oct 2021 18:04:53 +0300
Subject: [PATCH 009/102] utils: use fetch for `getUriStatus`

---
 lib/utils.js | 69 ++++++++++++++++++++++++----------------------------
 1 file changed, 32 insertions(+), 37 deletions(-)

diff --git a/lib/utils.js b/lib/utils.js
index db115fea2..9c0ce13fc 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -966,18 +966,17 @@ export function sendLogToWhitelist(uri, context) {
             data.oembed = oembedHref;
         }
 
-        request({
-            uri: CONFIG.WHITELIST_LOG_URL,
+        fetch(CONFIG.WHITELIST_LOG_URL, {
             method: 'GET',
             qs: data
         })
-            .on('error', function(error) {
-                console.error('Error logging url:', uri, error);
-            })
-            .on('response', function(res) {
-                if (res.statusCode !== 200) {
-                    console.error('Error logging url:', uri, res.statusCode);
+            .then(res => {
+                if (res.status !== 200) {
+                    console.error('Error logging url:', uri, res.status);
                 }
+            })
+            .catch(error => {
+                console.error('Error logging url:', uri, error);
             });
     }
 };
@@ -1135,33 +1134,38 @@ export function generateLinksHtml(data, options) {
 
 var getUriStatusPrivate = function(uri, options, cb) {
 
+    const controller = new AbortController();
+
+    var request_options = prepareRequestOptions({
+        uri: uri,
+        method: 'GET',
+        headers: {
+            'User-Agent': CONFIG.USER_AGENT
+        },
+        maxRedirects: CONFIG.MAX_REDIRECTS,
+        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+        jar: request.jar(),   //Enable cookies, uses new jar
+        signal: controller.signal
+    }, {
+        disableHttp2: options.disableHttp2
+    })
+
     try {
-        var r = request(prepareRequestOptions({
-            uri: uri,
-            method: 'GET',
-            headers: {
-                'User-Agent': CONFIG.USER_AGENT
-            },
-            maxRedirects: CONFIG.MAX_REDIRECTS,
-            timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-            jar: request.jar()   //Enable cookies, uses new jar
-        }, {
-            disableHttp2: options.disableHttp2
-        }))
-            .on('error', cb)
-            .on('response', function(res) {
-                abortRequest(r);
+        fetch(request_options.uri || request_options.url, request_options)
+            .then(res => {
+                controller.abort();
+
                 var data = {
-                    code: res.statusCode,
-                    content_type: res.headers && res.headers['content-type'],
-                    content_length: res.headers && res.headers['content-length'] ? parseInt(res.headers['content-length'] || '0', 10) : null
+                    code: res.status,
+                    content_type: res.headers && res.headers.get('content-type'),
+                    content_length: res.headers && res.headers.get('content-length') ? parseInt(res.headers.get('content-length') || '0', 10) : null
                 };
                 if (options.checkHeaders) {
                     data.headers = res.headers.raw();
                 }
                 cb(null, data);
-            });
-
+            })
+            .catch(cb)
     } catch (ex) {
         cb(ex.message);
     }
@@ -1302,15 +1306,6 @@ export function getIframelyErrorShortCode(error) {
     return error.code;
 };
 
-export function abortRequest(request) {
-    if (request && request.response && request.response.h2) {
-        // Abort http2 with special method.
-        request.response.abort();
-    } else if (request && !request.aborted) {
-        request.abort();
-    }
-};
-
 /**
  * @public
  * Generate slug for provider based on domain name of URL provided

From e3f57f44f52fc72af4e0d8bade2e6ce457c506d0 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 14 Oct 2021 18:18:33 +0300
Subject: [PATCH 010/102] Update config.local.js.SAMPLE

---
 config.local.js.SAMPLE | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/config.local.js.SAMPLE b/config.local.js.SAMPLE
index 88a8cf8e6..61dac1ba2 100644
--- a/config.local.js.SAMPLE
+++ b/config.local.js.SAMPLE
@@ -1,3 +1,9 @@
+    import { fileURLToPath } from 'url';
+    import { dirname } from 'path';
+
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+
     export default {
 
         // Specify a path for custom plugins. Custom plugins will override core plugins.

From 3fa0fd81f906947fc8ed85a917e848fd34208d9e Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 16:21:35 +0300
Subject: [PATCH 011/102] bugfix import export

---
 app.js                                                      | 6 ++++--
 .../system/htmlparser/CollectingHandlerForMutliTarget.js    | 3 ++-
 lib/plugins/system/oembed/oembedUtils.js                    | 5 ++---
 modules/tests-ui/models.js                                  | 4 ++--
 modules/tests-ui/utils.js                                   | 4 ++--
 modules/tests-ui/views.js                                   | 3 +--
 server.js                                                   | 2 +-
 7 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/app.js b/app.js
index 2718ae0ba..abf0549cc 100644
--- a/app.js
+++ b/app.js
@@ -8,12 +8,14 @@ if (!CONFIG.baseAppUrl) {
   console.warn('Warning: CONFIG.baseAppUrl not set, default value used');
 }
 
-import * as express from 'express';
+import express from 'express';
 import * as jsonxml from 'jsontoxml';
 
 var NotFound = sysUtils.NotFound;
 
-export const app = express();
+const app = express();
+
+export default app;
 
 app.set('view engine', 'ejs');
 
diff --git a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
index a9aa4ce54..6bf05a7fc 100644
--- a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
+++ b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
@@ -9,7 +9,8 @@ export function CollectingHandlerForMutliTarget(cbsArray){
     this._cbsArray = cbsArray || [];
     this.events = [];
 }
-import { EVENTS } from "htmlparser2";
+import htmlparser2 from "htmlparser2";
+const EVENTS = htmlparser2.EVENTS;
 
 Object.keys(EVENTS).forEach(function(name) {
 
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index 1e87a119c..cf3372cb2 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -13,6 +13,8 @@ var lowerCaseKeys = utils.lowerCaseKeys;
 
 export const notPlugin = true;
 
+const providers = JSON.parse(await readFile(new URL('./providers.json', import.meta.url)));
+
 /**
  * @private
  * Get the oembed uri via known providers
@@ -20,9 +22,6 @@ export const notPlugin = true;
  * @return {String} The oembed uri
  */
 function lookupStaticProviders(uri) {
-    
-    const providers = JSON.parse(await readFile(new URL('./providers.json', import.meta.url)));
-
     var protocolMatch = uri.match(/^(https?:\/\/)/);
     if (!protocolMatch || /^https?:\/\/blog\./i.test(uri)) {
         return null;
diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js
index 2c6320141..b11bcbfc4 100644
--- a/modules/tests-ui/models.js
+++ b/modules/tests-ui/models.js
@@ -4,7 +4,8 @@
 
     // DB connect.
     try {
-        mongoose = await import('mongoose');
+        const mongoosePkg = await import('mongoose');
+        mongoose = mongoosePkg.default;
         mongoose.set('useUnifiedTopology', true);
         mongoose.set('useCreateIndex', true);
         mongoose.set('useNewUrlParser', true);
@@ -15,7 +16,6 @@
     } catch (ex) {
         console.error("Plugins testing framework will not work. Can't connect to mongodb.");
         console.error(ex.stack);
-        return;
     }
 
     var Schema = mongoose.Schema;
diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
index c9445205b..236579daa 100644
--- a/modules/tests-ui/utils.js
+++ b/modules/tests-ui/utils.js
@@ -4,8 +4,8 @@ import * as request from 'request';
 import * as async from 'async';
 import * as url from 'url';
 import { PageTestLog, TestUrlsSet, PluginTest } from './models.js';
-import { findWhitelistRecordFor } from '../../lib/whitelist';
-import { getPluginData as iframelyGetPluginData } from '../../lib/core';
+import { findWhitelistRecordFor } from '../../lib/whitelist.js';
+import { getPluginData as iframelyGetPluginData } from '../../lib/core.js';
 import * as pluginLoader from '../../lib/loader/pluginLoader.js';
 import * as pluginUtils from '../../lib/loader/utils.js';
 
diff --git a/modules/tests-ui/views.js b/modules/tests-ui/views.js
index 6b01eedf2..d7785228a 100644
--- a/modules/tests-ui/views.js
+++ b/modules/tests-ui/views.js
@@ -150,5 +150,4 @@
                 });
             });
         });
-    }
-})();
+    }
\ No newline at end of file
diff --git a/server.js b/server.js
index cf5b8edd2..a9c8152dd 100644
--- a/server.js
+++ b/server.js
@@ -1,5 +1,5 @@
 import * as sysUtils from './utils.js';
-import * as app from './app.js';
+import app from './app.js';
 
 var server = app.listen(process.env.PORT || CONFIG.port, process.env.HOST || CONFIG.host, function(){
     console.log('\niframely is running on ' + server.address().address + ':' + server.address().port);

From e4e031bb044db7445453bcbe07f0f311da97a1b4 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 16:37:17 +0300
Subject: [PATCH 012/102] bugfix cache.js import export

---
 app.js                                          | 6 ++----
 lib/cache.js                                    | 4 ++--
 lib/plugins/system/htmlparser/htmlparser.js     | 2 +-
 lib/plugins/system/meta/cachedMeta.js           | 2 +-
 lib/plugins/system/meta/meta.js                 | 2 +-
 lib/plugins/system/oembed/oembedUtils.js        | 2 +-
 lib/plugins/validators/async/20_checkFavicon.js | 2 +-
 lib/plugins/validators/async/22_imageSize.js    | 2 +-
 lib/request.js                                  | 2 +-
 lib/utils.js                                    | 2 +-
 modules/api/views.js                            | 2 +-
 plugins/custom/domain-icon.js                   | 2 +-
 plugins/domains/twitter.com/twitter.status.js   | 2 +-
 utils.js                                        | 2 +-
 14 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/app.js b/app.js
index abf0549cc..94b4f5ad1 100644
--- a/app.js
+++ b/app.js
@@ -1,4 +1,4 @@
-import * as sysUtils from './utils.js';
+import { cacheMiddleware, NotFound } from './utils.js';
 
 console.log("");
 console.log("Starting Iframely...");
@@ -11,8 +11,6 @@ if (!CONFIG.baseAppUrl) {
 import express from 'express';
 import * as jsonxml from 'jsontoxml';
 
-var NotFound = sysUtils.NotFound;
-
 const app = express();
 
 export default app;
@@ -41,7 +39,7 @@ app.use(function(req, res, next) {
   next();
 });
 
-app.use(sysUtils.cacheMiddleware);
+app.use(cacheMiddleware);
 
 import apiViews from './modules/api/views.js';
 import debugViews from './modules/debug/views.js';
diff --git a/lib/cache.js b/lib/cache.js
index 04bbcfdec..b09ef18ef 100644
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -50,7 +50,7 @@
 
     await setCachingEngine(CONFIG.CACHE_ENGINE || DEFAULT_CACHE);
 
-    export function withCache(key, func, options, callback) {
+    cache.withCache = function(key, func, options, callback) {
 
         if (typeof options === 'function') {
             callback = options;
@@ -143,7 +143,7 @@
     *
     * */
 
-    export function MultiCache(options) {
+    const MultiCache = cache.MultiCache = function(options) {
         this.options = options;
 
         this.keysDict = {};
diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 8c7071579..e58abe699 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -2,7 +2,7 @@ import * as _ from 'underscore';
 import * as urlLib from 'url';
 import * as htmlparser2 from 'htmlparser2';
 var Parser = htmlparser2.Parser;
-import * as cache from '../../../cache.js';
+import { cache } from '../../../cache.js';
 import * as utils from '../../../utils.js';
 import * as libUtils from '../../../utils.js';
 import * as metaUtils from '../meta/utils.js';
diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js
index 74b9f9200..8fe48d622 100644
--- a/lib/plugins/system/meta/cachedMeta.js
+++ b/lib/plugins/system/meta/cachedMeta.js
@@ -1,5 +1,5 @@
 import * as urlLib from 'url';
-import * as cache from '../../../cache.js';
+import { cache } from '../../../cache.js';
 import * as sysUtils from '../../../../logging.js';
 import * as utils from './utils.js';
 import * as libUtils from '../../../utils.js';
diff --git a/lib/plugins/system/meta/meta.js b/lib/plugins/system/meta/meta.js
index ac72c3524..e352e450a 100644
--- a/lib/plugins/system/meta/meta.js
+++ b/lib/plugins/system/meta/meta.js
@@ -1,5 +1,5 @@
 import * as HTMLMetaHandler from './HTMLMetaHandler.js';
-import * as cache from '../../../cache.js';
+import { cache } from '../../../cache.js';
 import * as sysUtils from '../../../../logging.js';
 import * as libUtils from '../../../utils.js';
 import * as iconv from 'iconv-lite';
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index cf3372cb2..27fe68999 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -3,7 +3,7 @@ import * as urlLib from 'url';
 import * as async from 'async';
 import * as utils from '../../../utils.js';
 import * as sysUtils from '../../../../logging.js';
-import * as cache from '../../../cache.js';
+import { cache } from '../../../cache.js';
 import { readFile } from 'fs/promises';
 
 var getUrlFunctional = utils.getUrlFunctional;
diff --git a/lib/plugins/validators/async/20_checkFavicon.js b/lib/plugins/validators/async/20_checkFavicon.js
index 768359e37..6d095e783 100644
--- a/lib/plugins/validators/async/20_checkFavicon.js
+++ b/lib/plugins/validators/async/20_checkFavicon.js
@@ -1,7 +1,7 @@
 import * as urlLib from 'url';
 import * as _ from 'underscore';
 import * as utils from '../../../utils.js';
-import * as cache from '../../../cache.js';
+import { cache } from '../../../cache.js';
 
 export default {
 
diff --git a/lib/plugins/validators/async/22_imageSize.js b/lib/plugins/validators/async/22_imageSize.js
index 3cc66d446..2676b3e21 100644
--- a/lib/plugins/validators/async/22_imageSize.js
+++ b/lib/plugins/validators/async/22_imageSize.js
@@ -1,5 +1,5 @@
 import * as utils from '../../../utils.js';
-import * as cache from '../../../cache.js';
+import { cache } from '../../../cache.js';
 import * as mediaPlugin from '../media.js';
 
 
diff --git a/lib/request.js b/lib/request.js
index 41d88a1eb..9df061d88 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -1,7 +1,7 @@
 import * as request from 'request';
 import * as _ from 'underscore';
 import * as crypto from 'crypto';
-import * as cache from './cache.js';
+import { cache } from './cache.js';
 import * as utils from './utils.js';
 import * as sysUtils from '../logging.js';
 
diff --git a/lib/utils.js b/lib/utils.js
index 9c0ce13fc..5a99ccc25 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -12,7 +12,7 @@ import * as imagesize from 'probe-image-size';
 import * as _ from 'underscore';
 import * as parseIsoDuration from 'parse-iso-duration';
 import * as entities from 'entities';
-import * as cache from './cache.js';
+import { cache } from './cache.js';
 import * as htmlUtils from './html-utils.js';
 import * as sysUtils from '../logging.js';
 
diff --git a/modules/api/views.js b/modules/api/views.js
index eabb5f0cf..76ca0ac08 100644
--- a/modules/api/views.js
+++ b/modules/api/views.js
@@ -2,7 +2,7 @@ import * as iframelyCore from '../../lib/core.js';
 import * as utils from '../../utils.js';
 import * as _ from 'underscore';
 import * as async from 'async';
-import * as cache from '../../lib/cache.js';
+import { cache } from '../../lib/cache.js';
 import * as iframelyUtils from '../../lib/utils.js';
 import * as oembedUtils from '../../lib/oembed.js';
 import * as whitelist from '../../lib/whitelist.js';
diff --git a/plugins/custom/domain-icon.js b/plugins/custom/domain-icon.js
index 7dea9e587..b3bf4d6ac 100644
--- a/plugins/custom/domain-icon.js
+++ b/plugins/custom/domain-icon.js
@@ -1,6 +1,6 @@
 // use this mixin for domain plugins where you do not want to pull out htmlparser but do need an icon or logo
 import * as core from '../../lib/core.js';
-import * as cache from '../../lib/cache.js';
+import { cache } from '../../lib/cache.js';
 import * as async from 'async';
 import * as _ from 'underscore';
 import log from '../../logging.js';
diff --git a/plugins/domains/twitter.com/twitter.status.js b/plugins/domains/twitter.com/twitter.status.js
index a759ecd18..2f72f4d86 100644
--- a/plugins/domains/twitter.com/twitter.status.js
+++ b/plugins/domains/twitter.com/twitter.status.js
@@ -1,5 +1,5 @@
 import * as async from 'async';
-import * as cache from '../../../lib/cache.js';
+import { cache } from '../../../lib/cache.js';
 import * as sysUtils from '../../../logging.js';
 import * as _ from 'underscore';
 import * as entities from 'entities';
diff --git a/utils.js b/utils.js
index ae83c7fb2..45b61cf93 100644
--- a/utils.js
+++ b/utils.js
@@ -2,7 +2,7 @@
     global.CONFIG = config;
 
     import * as async from 'async';
-    import * as cache from './lib/cache.js';
+    import { cache } from './lib/cache.js';
     import * as ejs from 'ejs';
     import * as fs from 'fs';
     import * as crypto from 'crypto';

From 0c253dd3fea228950949e39c381696d6ab5b9d40 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 18:51:50 +0300
Subject: [PATCH 013/102] core: add `iframelyRun` param; add `templates`,
 `plugin` param for validators

---
 lib/core.js                                      | 9 ++++++---
 lib/loader/pluginLoader.js                       | 6 ++++++
 lib/loader/utils.js                              | 4 +++-
 lib/plugins/validators/sync/01_qa_rels.js        | 7 +------
 lib/plugins/validators/sync/08_render.js         | 7 ++-----
 plugins/custom/domain-icon.js                    | 5 ++---
 plugins/links/hosted/promo.js                    | 5 ++---
 plugins/links/prerender/prerender.js             | 5 ++---
 plugins/links/prerender/react-app-fb-fallback.js | 5 ++---
 9 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index e216b1d9f..992978f8b 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -9,11 +9,12 @@
     import * as pluginLoader from './loader/pluginLoader.js';
     import * as requestWrapper from './request.js';
 
-    var plugins = pluginLoader._plugins,
+    const plugins = pluginLoader._plugins,
         providedParamsDict = pluginLoader._providedParamsDict,
         pluginsList = pluginLoader._pluginsList,
         usedParamsDict = pluginLoader._usedParamsDict,
         postPluginsList = pluginLoader._postPluginsList,
+        templates = pluginLoader._templates,
         PLUGIN_METHODS = pluginUtils.PLUGIN_METHODS;
 
 
@@ -662,8 +663,9 @@
 
                 var plugin = postPluginsList[i];
                 var method = {
-                    pluginId: plugin.id,
+                    plugin: plugin,
                     name: 'prepareLink',
+                    template: templates,
                     handle: plugin.module['prepareLink']
                 };
                 var params = plugin.methods[method.name];
@@ -1484,7 +1486,8 @@
                 url: uri,
                 cb: true,
                 options: options,
-                request: requestWrapperWithParams
+                request: requestWrapperWithParams,
+                iframelyRun: run
             },
 
             pluginsContexts = {},
diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js
index 93be9a359..91a34c6f7 100644
--- a/lib/loader/pluginLoader.js
+++ b/lib/loader/pluginLoader.js
@@ -150,15 +150,21 @@
             var pluginDeclaration = {};
             var notPlugin = false;
 
+            var pluginLoadTimeout = setTimeout(() => {
+                console.error("-- Error: Timeout loading plugin " + pluginPath + ". Maybe recurring dependency.");
+            }, 1000);
+
             // Load plugin.
             try {
                 plugin = await import(pluginPath);
+                clearTimeout(pluginLoadTimeout);
                 if (plugin.notPlugin) {
                     notPlugin = true;
                 }
                 plugin = plugin && plugin.default;
             } catch(ex) {
                 console.error("Error loading plugin", pluginPath, ex);
+                clearTimeout(pluginLoadTimeout);
                 return;
             }
 
diff --git a/lib/loader/utils.js b/lib/loader/utils.js
index 1a465a3b3..178d06726 100644
--- a/lib/loader/utils.js
+++ b/lib/loader/utils.js
@@ -6,13 +6,15 @@
         "options",
         "request",
         "whitelistRecord",
+        "iframelyRun",
         "__readabilityEnabled"
     ];
 
     export const POST_PLUGIN_DEFAULT_PARAMS = [
         "link",
         "pluginContext",
-        "pluginId",
+        "plugin",
+        "templates",
         "iterationPluginContext"
     ];
 
diff --git a/lib/plugins/validators/sync/01_qa_rels.js b/lib/plugins/validators/sync/01_qa_rels.js
index 8b9a87424..65e02a797 100644
--- a/lib/plugins/validators/sync/01_qa_rels.js
+++ b/lib/plugins/validators/sync/01_qa_rels.js
@@ -1,17 +1,12 @@
-import * as pluginLoader from '../../../loader/pluginLoader.js',
-    plugins = pluginLoader._plugins;
-
 export default {
 
-    prepareLink: function(whitelistRecord, options, link, pluginId) {
+    prepareLink: function(whitelistRecord, options, link, plugin) {
 
         if (link.type === CONFIG.T.flash) {
             link.error = 'Adobe Flash Player is no longer supported';
             return;
         }        
 
-        var plugin = plugins[pluginId];
-
         if (plugin.domain || plugin.custom || (link.rel && (link.rel.indexOf(CONFIG.R.icon) > -1 || link.rel.indexOf(CONFIG.R.thumbnail) > -1))) {
             return;
         }
diff --git a/lib/plugins/validators/sync/08_render.js b/lib/plugins/validators/sync/08_render.js
index 5ed5a38d2..41276c74b 100644
--- a/lib/plugins/validators/sync/08_render.js
+++ b/lib/plugins/validators/sync/08_render.js
@@ -1,15 +1,12 @@
 import * as ejs from 'ejs';
 import * as fs from 'fs';
 
-import * as pluginLoader from '../../../loader/pluginLoader.js';
-var templates = pluginLoader._templates;
-
 export default {
-    prepareLink: function(link, pluginId) {
+    prepareLink: function(link, plugin, templates) {
 
         if (!link.href && !link.html && (link.template || link.template_context)) {
 
-            var template = link.template || pluginId;
+            var template = link.template || plugin.id;
 
             if (!(template in templates)) {
                 console.error("No template found: " + template);
diff --git a/plugins/custom/domain-icon.js b/plugins/custom/domain-icon.js
index b3bf4d6ac..f99571706 100644
--- a/plugins/custom/domain-icon.js
+++ b/plugins/custom/domain-icon.js
@@ -1,5 +1,4 @@
 // use this mixin for domain plugins where you do not want to pull out htmlparser but do need an icon or logo
-import * as core from '../../lib/core.js';
 import { cache } from '../../lib/cache.js';
 import * as async from 'async';
 import * as _ from 'underscore';
@@ -13,7 +12,7 @@ export default {
         return domain_icons;
     },
 
-    getData: function(url, cb, options) {
+    getData: function(url, iframelyRun, options, cb) {
 
         // find domain and protocol
         var domain, protocol;
@@ -71,7 +70,7 @@ export default {
                     // + run icons validation right away
 
                     // forceSyncCheck - ask 'checkFavicon' to check favicon this time before callback.
-                    core.run(domainUri, _.extend({}, options, {forceSyncCheck: true}), function(error, data) {
+                    iframelyRun(domainUri, _.extend({}, options, {forceSyncCheck: true}), function(error, data) {
 
                         var icons;
 
diff --git a/plugins/links/hosted/promo.js b/plugins/links/hosted/promo.js
index 9c9bc3aa5..ee0910de8 100644
--- a/plugins/links/hosted/promo.js
+++ b/plugins/links/hosted/promo.js
@@ -1,11 +1,10 @@
-import * as core from '../../../lib/core.js';
 import * as _ from 'underscore';
 
 export default {
 
     provides: 'self',
 
-    getData: function(url, __promoUri, options, cb) {
+    getData: function(url, __promoUri, iframelyRun, options, cb) {
 
         // __promoUri may be not string if no rel=promo need to be added
         // see theplatform plugin for example
@@ -24,7 +23,7 @@ export default {
         delete options2.promoUri;
         delete options2.jar;
 
-        core.run(promoUri, options2, function(error, data) {
+        iframelyRun(promoUri, options2, function(error, data) {
 
             var wrappedError = null;
 
diff --git a/plugins/links/prerender/prerender.js b/plugins/links/prerender/prerender.js
index d1472cc38..adc844c0c 100644
--- a/plugins/links/prerender/prerender.js
+++ b/plugins/links/prerender/prerender.js
@@ -1,4 +1,3 @@
-import * as core from '../../../lib/core.js';
 import * as utils from './utils.js';
 
 export default {
@@ -7,7 +6,7 @@ export default {
 
     provides: ['appUriData', 'whenPrerender'],
 
-    getData: function(url, __appFlag, options, meta, cb) {
+    getData: function(url, __appFlag, iframelyRun, options, meta, cb) {
 
         if (CONFIG.PRERENDER && CONFIG.PRERENDER_URL && options.user_agent === CONFIG.FB_USER_AGENT) {
 
@@ -17,7 +16,7 @@ export default {
                 refresh: true
             }};
 
-            core.run(prerenderUrl, options2, function(error, data) {
+            iframelyRun(prerenderUrl, options2, function(error, data) {
 
                 var title = data && data.meta && ((data.meta.og && data.meta.og.title) || (data.meta.twitter && data.meta.twitter.title) || data.meta.title || data.meta['html-title']);
 
diff --git a/plugins/links/prerender/react-app-fb-fallback.js b/plugins/links/prerender/react-app-fb-fallback.js
index 78acfc681..4202e07be 100644
--- a/plugins/links/prerender/react-app-fb-fallback.js
+++ b/plugins/links/prerender/react-app-fb-fallback.js
@@ -1,4 +1,3 @@
-import * as core from '../../../lib/core.js';
 import * as utils from './utils.js';
 
 export default {
@@ -7,7 +6,7 @@ export default {
 
     provides: ['appUriData', 'whenReact'],
 
-    getData: function(url, __appFlag, options, cb) {
+    getData: function(url, __appFlag, iframelyRun, options, cb) {
 
         if (options.user_agent === CONFIG.FB_USER_AGENT) {
             return cb();
@@ -19,7 +18,7 @@ export default {
             user_agent: CONFIG.FB_USER_AGENT
         }};
 
-        core.run(url, options2, function(error, data) {
+        iframelyRun(url, options2, function(error, data) {
 
             if (data && data.meta && utils.maybeApp(data.meta)) {
                 return cb({

From ae577e2717c2255106197190adb462821b60375e Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 18:55:50 +0300
Subject: [PATCH 014/102] fix using request import

---
 lib/core.js                                        | 2 +-
 lib/request.js                                     | 2 +-
 lib/utils.js                                       | 2 +-
 modules/tests-ui/utils.js                          | 2 +-
 plugins/domains/dailymotion.com/dailymotion.com.js | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index 992978f8b..f3edbb940 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -1,6 +1,6 @@
 
     import * as _ from 'underscore';
-    import * as request from 'request';
+    import request from 'request';
     import * as urlLib from 'url';
     import * as pluginUtils from './loader/utils.js';
     import * as utils from './utils.js';
diff --git a/lib/request.js b/lib/request.js
index 9df061d88..14c4664a3 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -1,4 +1,4 @@
-import * as request from 'request';
+import request from 'request';
 import * as _ from 'underscore';
 import * as crypto from 'crypto';
 import { cache } from './cache.js';
diff --git a/lib/utils.js b/lib/utils.js
index 5a99ccc25..c846c009f 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,5 +1,5 @@
 import * as events from 'events';
-import * as request from 'request';
+import request from 'request';
 import * as fetch from 'node-fetch';
 import * as AbortController from 'abort-controller';
 import { Http2Agent } from './agent-http2.js';
diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
index 236579daa..b4684ddaf 100644
--- a/modules/tests-ui/utils.js
+++ b/modules/tests-ui/utils.js
@@ -1,6 +1,6 @@
 import * as _ from 'underscore';
 import * as FeedParser from 'feedparser';
-import * as request from 'request';
+import request from 'request';
 import * as async from 'async';
 import * as url from 'url';
 import { PageTestLog, TestUrlsSet, PluginTest } from './models.js';
diff --git a/plugins/domains/dailymotion.com/dailymotion.com.js b/plugins/domains/dailymotion.com/dailymotion.com.js
index b4b6f1afd..e94590427 100644
--- a/plugins/domains/dailymotion.com/dailymotion.com.js
+++ b/plugins/domains/dailymotion.com/dailymotion.com.js
@@ -1,5 +1,5 @@
 import * as querystring from 'querystring';
-import * as request from 'request';
+import request from 'request';
 
 export default {
 

From 1a448b2c3eff6b3ef952fc14796184b40a85cf0c Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 19:37:09 +0300
Subject: [PATCH 015/102] fix import `logging.js`

---
 lib/cache-engines/memcached.js                     | 12 ++++++------
 lib/cache-engines/redis.js                         |  6 +++---
 lib/core.js                                        | 14 +++++++-------
 .../system/htmlparser/nonHtmlContentData.js        |  4 ++--
 lib/plugins/system/meta/cachedMeta.js              | 12 ++++++------
 lib/plugins/system/meta/ld-json.js                 |  4 ++--
 lib/plugins/system/meta/meta.js                    |  6 +++---
 lib/plugins/system/oembed/oembedUtils.js           |  8 ++++----
 lib/request.js                                     |  6 +++---
 lib/utils.js                                       | 10 +++++-----
 plugins/custom/fb-error.js                         |  4 ++--
 plugins/domains/twitter.com/twitter.status.js      | 10 +++++-----
 plugins/domains/youtube.com/youtube.channel.js     |  4 ++--
 plugins/domains/youtube.com/youtube.video.js       |  4 ++--
 14 files changed, 52 insertions(+), 52 deletions(-)

diff --git a/lib/cache-engines/memcached.js b/lib/cache-engines/memcached.js
index 3cb04fa1b..f9c5ab9e3 100644
--- a/lib/cache-engines/memcached.js
+++ b/lib/cache-engines/memcached.js
@@ -1,5 +1,5 @@
 
-    import * as sysUtils from '../../logging.js';
+    import log from '../../logging.js';
     import * as crypto from 'crypto';
     import * as Memcached from 'memcached';
     import * as _ from 'underscore';
@@ -65,13 +65,13 @@
 
         if (storedData && storedData.length > MEMCACHED_MAX_DATA_SIZE) {
 
-            sysUtils.log('   -- Memcached handled set error ' + _key + ': The length of the value is ' + storedData.length + ' and greater than ' + MEMCACHED_MAX_DATA_SIZE);
+            log('   -- Memcached handled set error ' + _key + ': The length of the value is ' + storedData.length + ' and greater than ' + MEMCACHED_MAX_DATA_SIZE);
 
         } else {
 
             memcached.set(key, storedData, options && options.ttl || CONFIG.CACHE_TTL, function(error){
                 if (error) {
-                    sysUtils.log('   -- Memcached set error ' + _key + ' ' + error);
+                    log('   -- Memcached set error ' + _key + ' ' + error);
                 }
             });
         }
@@ -89,7 +89,7 @@
                     return;
                 }
                 finished = true;
-                sysUtils.log('   -- Memcached soft timeout on get ' + _key);
+                log('   -- Memcached soft timeout on get ' + _key);
                 cb(null, null);
             }, timeout);
         }
@@ -104,7 +104,7 @@
             clearTimeout(timeoutId);
 
             if (error) {
-                sysUtils.log('   -- Memcached get error ' + _key + ' ' + error);
+                log('   -- Memcached get error ' + _key + ' ' + error);
                 // Fail silent.
                 return cb(null, null);
             }
@@ -116,7 +116,7 @@
             try {
                 var parsedData = JSON.parse(data);
             } catch(ex) {
-                sysUtils.log('   -- Memcached: error JSON parse value ' + _key + ' ' + ex.message);
+                log('   -- Memcached: error JSON parse value ' + _key + ' ' + ex.message);
                 return cb(null, null);
             }
 
diff --git a/lib/cache-engines/redis.js b/lib/cache-engines/redis.js
index 68c42d956..a4932e8e2 100644
--- a/lib/cache-engines/redis.js
+++ b/lib/cache-engines/redis.js
@@ -1,4 +1,4 @@
-    import * as sysUtils from '../../logging.js';
+    import log from '../../logging.js';
 
     var client;
 
@@ -17,7 +17,7 @@
 
         multi.exec(function(error) {
             if (error) {
-                sysUtils.log('   -- Redis set error ' + key + ' ' + error);
+                log('   -- Redis set error ' + key + ' ' + error);
             }
         });
     };
@@ -27,7 +27,7 @@
         client.get(key, function(error, data) {
 
             if (error) {
-                sysUtils.log('   -- Redis get error ' + key + ' ' + error);
+                log('   -- Redis get error ' + key + ' ' + error);
                 return cb(null, null);
             }
 
diff --git a/lib/core.js b/lib/core.js
index f3edbb940..748b00a28 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -4,7 +4,7 @@
     import * as urlLib from 'url';
     import * as pluginUtils from './loader/utils.js';
     import * as utils from './utils.js';
-    import * as sysUtils from '../logging.js';
+    import log from '../logging.js';
     import * as oembedUtils from './oembed.js';
     import * as pluginLoader from './loader/pluginLoader.js';
     import * as requestWrapper from './request.js';
@@ -1266,10 +1266,10 @@
                 var r = result[i];
                 var redirect = r.error && r.error[SYS_ERRORS.redirect];
                 if (typeof redirect === "string") {
-                    sysUtils.log('   -- plugin redirect (by "' + r.method.pluginId + '")', redirect);
+                    log('   -- plugin redirect (by "' + r.method.pluginId + '")', redirect);
                     return redirect;
                 } else if (redirect && typeof redirect !== "string") {
-                    sysUtils.log('   -- skip plugin redirect, not string (by "' + r.method.pluginId + '")', redirect);
+                    log('   -- skip plugin redirect, not string (by "' + r.method.pluginId + '")', redirect);
                 }
             }
         }
@@ -1288,17 +1288,17 @@
                     // error - if error is responseCode (from oembedError).
                     var message = error.message || error;
                     
-                    sysUtils.log('   -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: message, uri: uri}));
+                    log('   -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: message, uri: uri}));
                     return error;
                 }
 
                 if (r.error && r.error[SYS_ERRORS.responseStatusCode]) {
-                    sysUtils.log('   -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: r.error[SYS_ERRORS.responseStatusCode], uri: uri}));
+                    log('   -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: r.error[SYS_ERRORS.responseStatusCode], uri: uri}));
                     return r.error[SYS_ERRORS.responseStatusCode];
                 }
 
                 if (r.error && r.error === SYS_ERRORS.timeout) {
-                    sysUtils.log('   -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: SYS_ERRORS.timeout, uri: uri}));
+                    log('   -- plugin response:', JSON.stringify({plugin: r.method.pluginId, response: SYS_ERRORS.timeout, uri: uri}));
                     return SYS_ERRORS.timeout;
                 }
             }
@@ -1636,7 +1636,7 @@
                     // If no data from domain plugins - try fallback to generic plugins.
                     if (!options.mixAllWithDomainPlugin && isDomainPluginsMode && !resultsHasDomainData(requiredPlugins, allResults.allData)) {
 
-                        sysUtils.log('   -- fallback from domain to generic', usedDomains, uri);
+                        log('   -- fallback from domain to generic', usedDomains, uri);
 
                         if (options.skipFallbackNotification) {
                             delete options.skipFallbackNotification;
diff --git a/lib/plugins/system/htmlparser/nonHtmlContentData.js b/lib/plugins/system/htmlparser/nonHtmlContentData.js
index 9c5e58356..efcc69793 100644
--- a/lib/plugins/system/htmlparser/nonHtmlContentData.js
+++ b/lib/plugins/system/htmlparser/nonHtmlContentData.js
@@ -1,4 +1,4 @@
-import * as sysUtils from '../../../../logging.js';
+import log from '../../../../logging.js';
 
 export default {
 
@@ -15,7 +15,7 @@ export default {
         // HEADS UP: do not ever remove the below check for 'javascript' or 'flash' in content type
         // if left allowed, it'll make apps vulnerable for XSS attacks as such files will be rendered as regular embeds
         if (/javascript|flash|application\/json|text\/xml|application\/xml/i.test(nonHtmlContentType)) {
-            sysUtils.log('   -- Non html content type: "' + nonHtmlContentType + '" for ' + url);
+            log('   -- Non html content type: "' + nonHtmlContentType + '" for ' + url);
             return cb({
                 responseStatusCode: 415
             });
diff --git a/lib/plugins/system/meta/cachedMeta.js b/lib/plugins/system/meta/cachedMeta.js
index 8fe48d622..968c5979a 100644
--- a/lib/plugins/system/meta/cachedMeta.js
+++ b/lib/plugins/system/meta/cachedMeta.js
@@ -1,6 +1,6 @@
 import * as urlLib from 'url';
 import { cache } from '../../../cache.js';
-import * as sysUtils from '../../../../logging.js';
+import log from '../../../../logging.js';
 import * as utils from './utils.js';
 import * as libUtils from '../../../utils.js';
 
@@ -35,7 +35,7 @@ export default {
             function callback(error, data) {
 
                 if (error) {
-                    sysUtils.log('   -- Error loading cached meta for: ' + url + '. ' + error);
+                    log('   -- Error loading cached meta for: ' + url + '. ' + error);
                 }
 
                 function noCacheFound() {
@@ -56,7 +56,7 @@ export default {
                             var record_age_sec = (new Date().getTime() / 1000) - data._sys_created_at;
                             if (record_age_sec > ttl) {
                                 // Ignore cache older then requested ttl.
-                                sysUtils.log('   -- Disable using old cache for: ' + url);
+                                log('   -- Disable using old cache for: ' + url);
                                 return noCacheFound();
                             }
                         }
@@ -75,13 +75,13 @@ export default {
                         }
 
                         // If data object has error attribute - return as error (e.g. redirect).
-                        sysUtils.log('   -- Using cached htmlparser error for: ' + url);
+                        log('   -- Using cached htmlparser error for: ' + url);
                         cb(data.error);
                     } else if (data.htmlparser) {
-                        sysUtils.log('   -- Using cached htmlparser data for: ' + url);
+                        log('   -- Using cached htmlparser data for: ' + url);
                         cb(null, data.htmlparser);
                     } else {
-                        sysUtils.log('   -- Using cached meta for: ' + url);
+                        log('   -- Using cached meta for: ' + url);
                         cb(null, {
                             meta: data,
                             __hasCachedMeta: true,
diff --git a/lib/plugins/system/meta/ld-json.js b/lib/plugins/system/meta/ld-json.js
index 9984702b7..4f7c0ae7c 100644
--- a/lib/plugins/system/meta/ld-json.js
+++ b/lib/plugins/system/meta/ld-json.js
@@ -1,4 +1,4 @@
-import * as sysUtils from '../../../../logging.js';
+import log from '../../../../logging.js';
 import * as utils from '../../../utils.js';
 
 export function ldParser(result, decode, uri) {
@@ -40,7 +40,7 @@ export function ldParser(result, decode, uri) {
                 
             } catch (ex) {
 
-                sysUtils.log('   -- Error parsing ld-json', uri, ex.message);
+                log('   -- Error parsing ld-json', uri, ex.message);
 
             }
         }
diff --git a/lib/plugins/system/meta/meta.js b/lib/plugins/system/meta/meta.js
index e352e450a..4c10beaae 100644
--- a/lib/plugins/system/meta/meta.js
+++ b/lib/plugins/system/meta/meta.js
@@ -1,6 +1,6 @@
 import * as HTMLMetaHandler from './HTMLMetaHandler.js';
 import { cache } from '../../../cache.js';
-import * as sysUtils from '../../../../logging.js';
+import log from '../../../../logging.js';
 import * as libUtils from '../../../utils.js';
 import * as iconv from 'iconv-lite';
 import * as utils from './utils.js';
@@ -27,7 +27,7 @@ export default {
                 // https://github.com/ashtuchkin/iconv-lite/issues/60
 
                 if (!iconv.encodingExists(meta.charset)) {
-                    sysUtils.log('   -- Unsupported encoding: ' + meta.charset + ' in ' + url);
+                    log('   -- Unsupported encoding: ' + meta.charset + ' in ' + url);
                     return cb({
                         responseStatusCode: 415
                     });
@@ -46,7 +46,7 @@ export default {
                     });
 
                     if (options.refresh) {
-                        sysUtils.log('   -- Refreshed meta cache for: ' + url);
+                        log('   -- Refreshed meta cache for: ' + url);
                     }
                 }
 
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index 27fe68999..197fc6a76 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -2,7 +2,7 @@ import * as sax from 'sax';
 import * as urlLib from 'url';
 import * as async from 'async';
 import * as utils from '../../../utils.js';
-import * as sysUtils from '../../../../logging.js';
+import log from '../../../../logging.js';
 import { cache } from '../../../cache.js';
 import { readFile } from 'fs/promises';
 
@@ -181,7 +181,7 @@ export function findOembedLinks(uri, meta) {
             // Fallback to `request` format. Remove after 10.10.2020.
             var cachedError = (data && data.response && data.data && data.data.error) || (data && data.error);
             if (cachedError) {
-                sysUtils.log('   -- Oembed cache error: ' + uri, JSON.stringify(data));
+                log('   -- Oembed cache error: ' + uri, JSON.stringify(data));
             }
 
             if (data && !cachedError) {
@@ -189,7 +189,7 @@ export function findOembedLinks(uri, meta) {
                     // Fallback to `request` format. Remove after 10.10.2020.
                     data = data.data;
                 }
-                sysUtils.log('   -- Using cached oembed for: ' + uri);
+                log('   -- Using cached oembed for: ' + uri);
                 return cb({good_data_from_cache: true}, data);
             }
 
@@ -354,7 +354,7 @@ function xmlStream2oembed(stream, callback) {
                     // Decode only non UTF-8 because sax makes decoding.
                     var oldValue = value;
                     value = encodeText(charset, value);
-                    sysUtils.log('      -- decode oembed xml (charset, in, out):', charset, oldValue, value);
+                    log('      -- decode oembed xml (charset, in, out):', charset, oldValue, value);
                 }
 
                 if (prop.match(/(width|height)$/)) {
diff --git a/lib/request.js b/lib/request.js
index 14c4664a3..579c65a51 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -3,7 +3,7 @@ import * as _ from 'underscore';
 import * as crypto from 'crypto';
 import { cache } from './cache.js';
 import * as utils from './utils.js';
-import * as sysUtils from '../logging.js';
+import log from '../logging.js';
 
 var hash = function(value) {
     return '"' + crypto.createHash('md5').update(value).digest("hex") + '"';
@@ -73,13 +73,13 @@ export default function(options, iframely_options, callback) {
 
                 // Retry on ECONNRESET.
                 if (!secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                    sysUtils.log('   -- request.js ' + error.code + ' first', url);
+                    log('   -- request.js ' + error.code + ' first', url);
                     process.nextTick(function() {
                         doRealRequest(true);
                     });
                     return;
                 } else if (secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                    sysUtils.log('   -- request.js ' + error.code + ' second', url);
+                    log('   -- request.js ' + error.code + ' second', url);
                 }
             }
 
diff --git a/lib/utils.js b/lib/utils.js
index c846c009f..2a739cd85 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -14,7 +14,7 @@ import * as parseIsoDuration from 'parse-iso-duration';
 import * as entities from 'entities';
 import { cache } from './cache.js';
 import * as htmlUtils from './html-utils.js';
-import * as sysUtils from '../logging.js';
+import log from '../logging.js';
 
 var http2Agent = new Http2Agent(/* options */);
 
@@ -144,7 +144,7 @@ export function getUrlFunctional(url, options, callbacks) {
 
             // Retry on ECONNRESET. Http2Agent will try with `http1` after this error.
             if (!options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                sysUtils.log('   -- getUrl ' + error.code + ' first', opts, url);
+                log('   -- getUrl ' + error.code + ' first', opts, url);
                 process.nextTick(function() {
                     getUrlFunctional(url, _.extend({}, options, {
                         secondAttempt: true, 
@@ -153,7 +153,7 @@ export function getUrlFunctional(url, options, callbacks) {
                 });
                 return;
             } else if (options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                sysUtils.log('   -- getUrl ' + error.code + ' second', opts, url);
+                log('   -- getUrl ' + error.code + ' second', opts, url);
             }
 
             callbacks.onError && callbacks.onError(error);
@@ -293,13 +293,13 @@ export function getHeadFunctional(url, options, callbacks) {
 
             // Retry on ECONNRESET.
             if (!options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                sysUtils.log('   -- getHead ' + error.code + ' first', url);
+                log('   -- getHead ' + error.code + ' first', url);
                 process.nextTick(function() {
                     getHeadFunctional(url, _.extend({}, options, {secondAttempt: true}), callbacks);
                 });
                 return;
             } else if (options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                sysUtils.log('   -- getHead ' + error.code + ' second', url);
+                log('   -- getHead ' + error.code + ' second', url);
             }
 
             callbacks.onError && callbacks.onError(error);
diff --git a/plugins/custom/fb-error.js b/plugins/custom/fb-error.js
index 5d298fe4b..89620c5ff 100644
--- a/plugins/custom/fb-error.js
+++ b/plugins/custom/fb-error.js
@@ -1,4 +1,4 @@
-import * as logging from '../../logging.js';
+import log from '../../logging.js';
 
 export default {
 
@@ -49,7 +49,7 @@ export default {
             result.message = fbError.message;
         }
 
-        logging.log('Facebook oembed api - error getting oembed for', url, JSON.stringify(fbError), JSON.stringify(result));
+        log('Facebook oembed api - error getting oembed for', url, JSON.stringify(fbError), JSON.stringify(result));
 
         return cb(result);
     },
diff --git a/plugins/domains/twitter.com/twitter.status.js b/plugins/domains/twitter.com/twitter.status.js
index 2f72f4d86..21abc6435 100644
--- a/plugins/domains/twitter.com/twitter.status.js
+++ b/plugins/domains/twitter.com/twitter.status.js
@@ -1,6 +1,6 @@
 import * as async from 'async';
 import { cache } from '../../../lib/cache.js';
-import * as sysUtils from '../../../logging.js';
+import log from '../../../logging.js';
 import * as _ from 'underscore';
 import * as entities from 'entities';
 
@@ -85,9 +85,9 @@ export default {
 
                         if (response.fromRequestCache) {
                             if (blockExpireIn > 0) {
-                                sysUtils.log('   -- Twitter API limit reached (' + blockExpireIn + ' seconds left), but cache used.');
+                                log('   -- Twitter API limit reached (' + blockExpireIn + ' seconds left), but cache used.');
                             } else {
-                                sysUtils.log('   -- Twitter API cache used.');
+                                log('   -- Twitter API cache used.');
                             }
                         }
 
@@ -113,9 +113,9 @@ export default {
                                 }
 
                                 if (response.statusCode === 429) {
-                                    sysUtils.log('   -- Twitter API limit reached by status code 429. Disabling for ' + ttl + ' seconds.');
+                                    log('   -- Twitter API limit reached by status code 429. Disabling for ' + ttl + ' seconds.');
                                 } else {
-                                    sysUtils.log('   -- Twitter API limit warning, remaining calls: ' + remaining + '. Disabling for ' + ttl + ' seconds.');
+                                    log('   -- Twitter API limit warning, remaining calls: ' + remaining + '. Disabling for ' + ttl + ' seconds.');
                                 }
 
                                 // Store expire date as value to be sure it past.
diff --git a/plugins/domains/youtube.com/youtube.channel.js b/plugins/domains/youtube.com/youtube.channel.js
index 18702364f..8055eed56 100644
--- a/plugins/domains/youtube.com/youtube.channel.js
+++ b/plugins/domains/youtube.com/youtube.channel.js
@@ -1,4 +1,4 @@
-import * as sysUtils from '../../../logging.js'
+import log from '../../../logging.js'
 
 export default {
 
@@ -90,7 +90,7 @@ export default {
                      * as if no channel found, but it actually exists.
                      * Ex.: https://www.youtube.com/c/Figmadesign
                      */
-                    sysUtils.log('YoutTube channel fallback for ' + url , data);
+                    log('YoutTube channel fallback for ' + url , data);
                     cb({
                         message: 'YouTube channel not found via data API...'
                         // But no response code. Let's fallback to default parsers
diff --git a/plugins/domains/youtube.com/youtube.video.js b/plugins/domains/youtube.com/youtube.video.js
index 6487a1201..d9e9863ea 100644
--- a/plugins/domains/youtube.com/youtube.video.js
+++ b/plugins/domains/youtube.com/youtube.video.js
@@ -1,7 +1,7 @@
 import * as cheerio from 'cheerio';
 import * as querystring from 'querystring';
 import * as _ from 'underscore';
-import * as sysUtils from '../../../logging.js'
+import log from '../../../logging.js'
 
 export default {
 
@@ -129,7 +129,7 @@ export default {
                 } else if (data.items && data.items.length == 0 || data.error && data.error.code == 404) {
                     cb({responseStatusCode: 404});
                 } else {
-                    sysUtils.log('YoutTube fallback for ' + urlMatch[1], data);
+                    log('YoutTube fallback for ' + urlMatch[1], data);
                     cb(null); // silence error for fallback to generic providers. data.error.code == 429 - too many requests; 400 - probably API key is invalid
                 }
             }

From a73348bbcd1ed7497a58fd7ba8c7c72d68e9ad72 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 19:45:25 +0300
Subject: [PATCH 016/102] core: add `templates`, `plugin` param for validators

---
 lib/core.js | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/core.js b/lib/core.js
index 748b00a28..c2217d569 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -663,6 +663,7 @@
 
                 var plugin = postPluginsList[i];
                 var method = {
+                    pluginId: plugin.id,
                     plugin: plugin,
                     name: 'prepareLink',
                     template: templates,
@@ -752,9 +753,19 @@
 
                     } else if (param === 'pluginId') {
 
-                        args.push(dataRecord.method.pluginId);
+                        
+                    } else if (param === 'plugin') {
+
+                        args.push(dataRecord.method.plugin);
+
+                        continue;
+
+                    } else if (param === 'templates') {
+
+                        args.push(templates);
 
                         continue;
+                        
                     }
 
                     args.push(context[param]);

From 709ac76d5478e62d1b5df4199bab9e52aded3f8a Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 19:45:49 +0300
Subject: [PATCH 017/102] remove unused

---
 lib/core.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/core.js b/lib/core.js
index c2217d569..012a8a591 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -666,7 +666,6 @@
                     pluginId: plugin.id,
                     plugin: plugin,
                     name: 'prepareLink',
-                    template: templates,
                     handle: plugin.module['prepareLink']
                 };
                 var params = plugin.methods[method.name];

From ac5b0cbeba23236a4041fbd568f5c0b810a335c4 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 19:46:50 +0300
Subject: [PATCH 018/102] fix typo

---
 lib/loader/pluginLoader.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js
index 91a34c6f7..0864b0bf7 100644
--- a/lib/loader/pluginLoader.js
+++ b/lib/loader/pluginLoader.js
@@ -25,7 +25,7 @@
 
     export {
         plugins as _plugins,
-        providedParamsDict as _p_providedParamsDictlugins,
+        providedParamsDict as _providedParamsDict,
         usedParamsDict as _usedParamsDict,
         templates as _templates,
         pluginsList as _pluginsList,

From 2f317b810b6284eb3096c4a752832fe38edb84ba Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 19:47:34 +0300
Subject: [PATCH 019/102] bugfix abort

---
 lib/utils.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/utils.js b/lib/utils.js
index 2a739cd85..a2499a3bb 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -716,7 +716,8 @@ export function getContentType(uriForCache, uriOriginal, options, cb) {
             }
 
             // We don't need more data. Abort causes error. timeout === null here so error will be skipped.
-            abortController.abort();
+            // If 'abortController' not defined, then no request created?
+            abortController && abortController.abort();
 
             var data = {};
 

From 0d858b26fe3a34f50c8b6400f047a1c265f54eeb Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 20:00:37 +0300
Subject: [PATCH 020/102] core: fix `plugin` param for validators

---
 lib/core.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index 012a8a591..6190b878e 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -664,7 +664,6 @@
                 var plugin = postPluginsList[i];
                 var method = {
                     pluginId: plugin.id,
-                    plugin: plugin,
                     name: 'prepareLink',
                     handle: plugin.module['prepareLink']
                 };
@@ -750,12 +749,9 @@
 
                         continue;
 
-                    } else if (param === 'pluginId') {
-
-                        
                     } else if (param === 'plugin') {
 
-                        args.push(dataRecord.method.plugin);
+                        args.push(plugins[dataRecord.method.pluginId]);
 
                         continue;
 

From bd0a75d374941e704122506bb75417cf010ef4e9 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 20:00:58 +0300
Subject: [PATCH 021/102] validators: fix imports

---
 lib/plugins/validators/async/21_checkContentType.js   | 6 +++---
 lib/plugins/validators/async/22_imageSize.js          | 2 +-
 lib/plugins/validators/sync/02_html5_multimedia.js    | 2 +-
 lib/plugins/validators/sync/03_media.js               | 2 +-
 lib/plugins/validators/sync/12_player_no_scrolling.js | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/plugins/validators/async/21_checkContentType.js b/lib/plugins/validators/async/21_checkContentType.js
index 49058ec5b..5d01b1097 100644
--- a/lib/plugins/validators/async/21_checkContentType.js
+++ b/lib/plugins/validators/async/21_checkContentType.js
@@ -1,8 +1,8 @@
 import * as utils from '../../../utils.js';
 import * as urlLib from 'url';
-import * as multimedia from '../html5_multimedia.js';
-import * as mediaPlugin from '../media.js';
-import * as player_no_scrolling from '../player_no_scrolling.js';
+import multimedia from '../html5_multimedia.js';
+import mediaPlugin from '../media.js';
+import player_no_scrolling from '../player_no_scrolling.js';
 
 export default {
 
diff --git a/lib/plugins/validators/async/22_imageSize.js b/lib/plugins/validators/async/22_imageSize.js
index 2676b3e21..9ea36e8d0 100644
--- a/lib/plugins/validators/async/22_imageSize.js
+++ b/lib/plugins/validators/async/22_imageSize.js
@@ -1,6 +1,6 @@
 import * as utils from '../../../utils.js';
 import { cache } from '../../../cache.js';
-import * as mediaPlugin from '../media.js';
+import mediaPlugin from '../media.js';
 
 
 export default {
diff --git a/lib/plugins/validators/sync/02_html5_multimedia.js b/lib/plugins/validators/sync/02_html5_multimedia.js
index ad402ae31..d183dfc5b 100644
--- a/lib/plugins/validators/sync/02_html5_multimedia.js
+++ b/lib/plugins/validators/sync/02_html5_multimedia.js
@@ -1,4 +1,4 @@
-import * as multimedia from '../html5_multimedia.js';
+import multimedia from '../html5_multimedia.js';
 
 export default {
 
diff --git a/lib/plugins/validators/sync/03_media.js b/lib/plugins/validators/sync/03_media.js
index 338bd3d66..183cfae28 100644
--- a/lib/plugins/validators/sync/03_media.js
+++ b/lib/plugins/validators/sync/03_media.js
@@ -1,4 +1,4 @@
-import * as mediaPlugin from '../media.js';
+import mediaPlugin from '../media.js';
 
 export default {
 
diff --git a/lib/plugins/validators/sync/12_player_no_scrolling.js b/lib/plugins/validators/sync/12_player_no_scrolling.js
index 0ad6db321..494f19d8a 100644
--- a/lib/plugins/validators/sync/12_player_no_scrolling.js
+++ b/lib/plugins/validators/sync/12_player_no_scrolling.js
@@ -1,4 +1,4 @@
-import * as player_no_scrolling from '../player_no_scrolling.js';
+import player_no_scrolling from '../player_no_scrolling.js';
 
 export default {
 

From c7dc3e67d6c61084998062716ecbeeaea5cfe8e3 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 20:01:54 +0300
Subject: [PATCH 022/102] bugfixes for testrun

---
 lib/plugins/system/htmlparser/htmlparser.js | 3 ++-
 lib/utils.js                                | 9 ++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index e58abe699..f712289fe 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -86,7 +86,8 @@ export default {
                     }
                 }
 
-                var headers = resp.headers.raw();
+                // TODO: 
+                var headers = resp.headers.raw ? resp.headers.raw() : resp.headers;
 
                 if (resp.status >= 300 && resp.status< 400 && headers.location) {
                     abortController.abort();
diff --git a/lib/utils.js b/lib/utils.js
index a2499a3bb..6d5bf8e80 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,7 +1,7 @@
 import * as events from 'events';
 import request from 'request';
-import * as fetch from 'node-fetch';
-import * as AbortController from 'abort-controller';
+import fetch from 'node-fetch';
+import AbortController from 'abort-controller';
 import { Http2Agent } from './agent-http2.js';
 import * as zlib from 'zlib';
 import iltorb from 'iltorb';
@@ -218,6 +218,9 @@ export function getUrlFunctional(url, options, callbacks) {
                     var headers = res.headers.raw();
 
                     var contentEncoding = headers['content-encoding'];
+                    if (Array.isArray(contentEncoding)) {
+                        contentEncoding = contentEncoding[0];
+                    }
                     contentEncoding = contentEncoding && contentEncoding.trim().toLowerCase();
 
                     var zlibOptions = {
@@ -515,7 +518,7 @@ export function parseJSONSource(text, decode) {
             }
 
             // We don't need more data. Abort causes error. timeout === null here so error will be skipped.
-            abortController.abort();
+            abortController && abortController.abort();
 
             if (!error && !data) {
                 error = 404;

From f26ca312b4c38a30637f4a7d331c64e3e1e519b6 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 22:10:54 +0300
Subject: [PATCH 023/102] fix plugins utils import

---
 plugins/links/iframely-link.js                   | 2 +-
 plugins/links/image_src.js                       | 2 +-
 plugins/links/prerender/checkAppFlag.js          | 2 +-
 plugins/links/prerender/prerender.js             | 2 +-
 plugins/links/prerender/react-app-fb-fallback.js | 2 +-
 plugins/links/sailthru.js                        | 2 +-
 plugins/links/thumbnail.js                       | 2 +-
 plugins/meta/media-detector.js                   | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/plugins/links/iframely-link.js b/plugins/links/iframely-link.js
index fd2798050..36cc1231d 100644
--- a/plugins/links/iframely-link.js
+++ b/plugins/links/iframely-link.js
@@ -1,4 +1,4 @@
-import * as utils from './utils.js';
+import utils from './utils.js';
 import * as _ from 'underscore';
 
 export default {
diff --git a/plugins/links/image_src.js b/plugins/links/image_src.js
index 2655c9145..af29792bf 100644
--- a/plugins/links/image_src.js
+++ b/plugins/links/image_src.js
@@ -1,4 +1,4 @@
-import * as utils from './utils.js';
+import utils from './utils.js';
 
 export default {
 
diff --git a/plugins/links/prerender/checkAppFlag.js b/plugins/links/prerender/checkAppFlag.js
index cc35d48c3..2748ac785 100644
--- a/plugins/links/prerender/checkAppFlag.js
+++ b/plugins/links/prerender/checkAppFlag.js
@@ -1,4 +1,4 @@
-import * as utils from './utils.js';
+import utils from './utils.js';
 
 export default {
 
diff --git a/plugins/links/prerender/prerender.js b/plugins/links/prerender/prerender.js
index adc844c0c..7bd0f9102 100644
--- a/plugins/links/prerender/prerender.js
+++ b/plugins/links/prerender/prerender.js
@@ -1,4 +1,4 @@
-import * as utils from './utils.js';
+import utils from './utils.js';
 
 export default {
 
diff --git a/plugins/links/prerender/react-app-fb-fallback.js b/plugins/links/prerender/react-app-fb-fallback.js
index 4202e07be..03b3ed171 100644
--- a/plugins/links/prerender/react-app-fb-fallback.js
+++ b/plugins/links/prerender/react-app-fb-fallback.js
@@ -1,4 +1,4 @@
-import * as utils from './utils.js';
+import utils from './utils.js';
 
 export default {
 
diff --git a/plugins/links/sailthru.js b/plugins/links/sailthru.js
index 77e381b85..8c72e8696 100644
--- a/plugins/links/sailthru.js
+++ b/plugins/links/sailthru.js
@@ -1,4 +1,4 @@
-import * as utils from './utils.js';
+import utils from './utils.js';
 
 export default {
 
diff --git a/plugins/links/thumbnail.js b/plugins/links/thumbnail.js
index 4793e73d3..604f517d5 100644
--- a/plugins/links/thumbnail.js
+++ b/plugins/links/thumbnail.js
@@ -1,4 +1,4 @@
-import * as utils from './utils.js';
+import utils from './utils.js';
 
 export default {
 
diff --git a/plugins/meta/media-detector.js b/plugins/meta/media-detector.js
index 8b1e9990e..0f8e5ffe3 100644
--- a/plugins/meta/media-detector.js
+++ b/plugins/meta/media-detector.js
@@ -1,4 +1,4 @@
-import * as utils from '../links/utils.js';
+import utils from '../links/utils.js';
 
 export default {
 

From fcb4a79ffc516d689114b7431de9915384dd9857 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 15 Oct 2021 22:11:20 +0300
Subject: [PATCH 024/102] bugfixes: fetch

---
 lib/plugins/system/htmlparser/htmlparser.js |  5 +-
 lib/plugins/system/meta/meta.js             |  4 +-
 lib/plugins/system/oembed/oembedUtils.js    |  6 +--
 lib/utils.js                                | 56 ++++++++++++++-------
 4 files changed, 46 insertions(+), 25 deletions(-)

diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index f712289fe..ae20b1068 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -7,7 +7,7 @@ import * as utils from '../../../utils.js';
 import * as libUtils from '../../../utils.js';
 import * as metaUtils from '../meta/utils.js';
 var getUrlFunctional = utils.getUrlFunctional;
-import * as CollectingHandlerForMutliTarget from './CollectingHandlerForMutliTarget.js';
+import { CollectingHandlerForMutliTarget } from './CollectingHandlerForMutliTarget.js';
 
 export default {
 
@@ -86,8 +86,7 @@ export default {
                     }
                 }
 
-                // TODO: 
-                var headers = resp.headers.raw ? resp.headers.raw() : resp.headers;
+                var headers = resp.headers;
 
                 if (resp.status >= 300 && resp.status< 400 && headers.location) {
                     abortController.abort();
diff --git a/lib/plugins/system/meta/meta.js b/lib/plugins/system/meta/meta.js
index 4c10beaae..6f404abfb 100644
--- a/lib/plugins/system/meta/meta.js
+++ b/lib/plugins/system/meta/meta.js
@@ -1,8 +1,8 @@
-import * as HTMLMetaHandler from './HTMLMetaHandler.js';
+import { HTMLMetaHandler } from './HTMLMetaHandler.js';
 import { cache } from '../../../cache.js';
 import log from '../../../../logging.js';
 import * as libUtils from '../../../utils.js';
-import * as iconv from 'iconv-lite';
+import iconv from 'iconv-lite';
 import * as utils from './utils.js';
 
 export default {
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index 197fc6a76..9078a2897 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -266,7 +266,7 @@ export function findOembedLinks(uri, meta) {
  * Convert XML or JSON stream to an oEmbed object.
  */
 function stream2oembed(uri, stream, callback) {
-    var contentType = stream.headers && stream.headers.get('content-type');
+    var contentType = stream.headers && stream.headers['content-type'];
     if (contentType && contentType.match('xml')) {
 
         xmlStream2oembed(stream, callback)
@@ -308,7 +308,7 @@ function xmlStream2oembed(stream, callback) {
     var value;
     var firstTag;
 
-    var charset = getCharset(stream.headers && stream.headers.get('content-type'));
+    var charset = getCharset(stream.headers && stream.headers['content-type']);
 
     var saxStream = sax.createStream();
 
@@ -382,7 +382,7 @@ function xmlStream2oembed(stream, callback) {
  */
 function jsonStream2oembed(stream, callback) {
 
-    var charset = getCharset(stream.headers && stream.headers.get('content-type'));
+    var charset = getCharset(stream.headers && stream.headers['content-type']);
 
     var data = "";
     stream.on('data', function(chunk) {
diff --git a/lib/utils.js b/lib/utils.js
index 6d5bf8e80..8331a3c03 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -49,6 +49,14 @@ function maxTTL(ttl) {
     return ttl;
 }
 
+function prepareFetchHeaders(headers) {
+    var result = {};
+    for(var key in headers) {
+        result[key] = headers[key][0];
+    }
+    return result;
+}
+
 export function getMaxCacheTTLOverride(url, options) {
     var proxy = null;
     if (CONFIG.PROXY || (options && options.proxy)) {
@@ -202,7 +210,9 @@ export function getUrlFunctional(url, options, callbacks) {
                 headers: {
                     'User-Agent': options.user_agent || CONFIG.USER_AGENT,
                     'Connection': 'keep-alive',                    
-                    'Accept-Encoding': 'gzip' + (sslProtocol ? ', br' : ''),
+                    // TODO: temp disable gzip
+                    'Accept-Encoding': 'text/html',
+                    //'Accept-Encoding': 'gzip' + (sslProtocol ? ', br' : ''),
                     'Accept': '*/*'
                 },
                 maxRedirects: options.maxRedirects || CONFIG.MAX_REDIRECTS,
@@ -215,7 +225,7 @@ export function getUrlFunctional(url, options, callbacks) {
             fetch(request_options.url || request_options.uri, request_options)
                 .then(res => {
 
-                    var headers = res.headers.raw();
+                    var headers = prepareFetchHeaders(res.headers.raw());
 
                     var contentEncoding = headers['content-encoding'];
                     if (Array.isArray(contentEncoding)) {
@@ -235,7 +245,7 @@ export function getUrlFunctional(url, options, callbacks) {
                         // TODO:
                         br.request = res.request;
                         // TODO:
-                        br.statusCode = res.status;
+                        br.status = res.status;
                         br.headers = headers;
 
                         if (!options.asBuffer) {
@@ -244,7 +254,7 @@ export function getUrlFunctional(url, options, callbacks) {
 
                         // TODO:
                         req.emit('response', br);
-                        res.pipe(br);
+                        res.body.pipe(br);
 
                     } else if (contentEncoding === 'gzip' || contentEncoding === 'deflate') {
 
@@ -253,7 +263,7 @@ export function getUrlFunctional(url, options, callbacks) {
                         // TODO:
                         gunzip.request = res.request;
                         // TODO:
-                        gunzip.statusCode = res.status;
+                        gunzip.status = res.status;
                         gunzip.headers = headers;
 
                         if (!options.asBuffer) {
@@ -262,16 +272,21 @@ export function getUrlFunctional(url, options, callbacks) {
 
                         // TODO:
                         req.emit('response', gunzip);
-                        res.pipe(gunzip);
+                        res.body.pipe(gunzip);
 
                     } else {
 
                         if (!options.asBuffer) {
-                            res.setEncoding("binary");
+                            // res.setEncoding("binary");
                         }
 
                         // TODO:
-                        req.emit('response', res);
+                        var body = res.body;
+                        body.status = res.status;
+                        body.headers = headers;
+
+                        // TODO:
+                        req.emit('response', body);
                     }
                 })
                 .catch(error => {
@@ -311,7 +326,12 @@ export function getHeadFunctional(url, options, callbacks) {
             callbacks.onAbortController && callbacks.onAbortController(abortController);
         })
         .on('response', function(response) {
-            callbacks.onResponse && callbacks.onResponse(response);
+            // TODO:
+            var body = response.body;
+            body.status = response.status;
+            body.headers = prepareFetchHeaders(response.headers.raw());
+
+            callbacks.onResponse && callbacks.onResponse(body);
         });
 }
 
@@ -377,7 +397,9 @@ export function getCharset(string, doNotParse) {
     if (doNotParse) {
         charset = string.toUpperCase();
     } else if (string) {
+        console.log('-111', string)
         var m = string && string.match(/charset\s*=\s*([\w_-]+)/i);
+        console.log('-222')
         charset = m && m[1].toUpperCase();
     }
 
@@ -596,7 +618,7 @@ export function parseJSONSource(text, decode) {
                     },
                     onResponse: function(res) {
 
-                        var content_type = res.headers.get('content-type');
+                        var content_type = res.headers['content-type'];
 
                         if (content_type && content_type !== 'application/octet-stream' && content_type !== 'binary/octet-stream') {
 
@@ -609,7 +631,7 @@ export function parseJSONSource(text, decode) {
                             if (options.debug) {
                                 imageResponseStarted = totalTime();
                             }
-                            contentLength = parseInt(res.headers.get('content-length') || '0', 10);
+                            contentLength = parseInt(res.headers['content-length'] || '0', 10);
                             imagesize(res, function(error, data) {
                                 if (data && data.type) {
                                     data.format = data.type;
@@ -788,8 +810,8 @@ export function getContentType(uriForCache, uriOriginal, options, cb) {
                             || error === 400 
                             || error >= 500
                             // Or ClourFront that gobbles up headers when checking CORS.
-                            || (res.headers && !res.headers.get('access-control-allow-origin')
-                                && res.headers.get('server') === 'AmazonS3' && !error ))) {
+                            || (res.headers && !res.headers['access-control-allow-origin']
+                                && res.headers['server'] === 'AmazonS3' && !error ))) {
                         makeCall('GET');
                         return;
                     }
@@ -800,7 +822,7 @@ export function getContentType(uriForCache, uriOriginal, options, cb) {
                         //res.headers.location = res.request.href;
                     }
 
-                    finish(error, res.headers.raw());
+                    finish(error, res.headers);
                 },
                 onError: function(error) {
                     finish(error);
@@ -1161,11 +1183,11 @@ var getUriStatusPrivate = function(uri, options, cb) {
 
                 var data = {
                     code: res.status,
-                    content_type: res.headers && res.headers.get('content-type'),
-                    content_length: res.headers && res.headers.get('content-length') ? parseInt(res.headers.get('content-length') || '0', 10) : null
+                    content_type: res.headers && res.headers['content-type'],
+                    content_length: res.headers && res.headers['content-length'] ? parseInt(res.headers['content-length'] || '0', 10) : null
                 };
                 if (options.checkHeaders) {
-                    data.headers = res.headers.raw();
+                    data.headers = prepareFetchHeaders(res.headers.raw());
                 }
                 cb(null, data);
             })

From 64447f00feff1c8a4cb8965cd9946ad9767dc371 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 18 Oct 2021 17:58:16 +0300
Subject: [PATCH 025/102] fix cheerio import

---
 lib/html-utils.js                                        | 2 +-
 lib/plugins/system/cheerio.js                            | 2 +-
 lib/plugins/system/oembed/oembed.js                      | 2 +-
 lib/utils.js                                             | 2 --
 plugins/domains/art19.com.js                             | 2 +-
 plugins/domains/brightcove.com/players.brightcove.net.js | 2 +-
 plugins/domains/cartodb.com.js                           | 2 +-
 plugins/domains/codepen.io.js                            | 2 +-
 plugins/domains/instagram.com/instagram.com.js           | 2 +-
 plugins/domains/mixcloud.com.js                          | 2 +-
 plugins/domains/scribd.com/scribd.com.js                 | 2 +-
 plugins/domains/slideshare.net.js                        | 2 +-
 plugins/domains/soundcloud.com/soundcloud.com.js         | 2 +-
 plugins/domains/speakerdeck.com.js                       | 2 +-
 plugins/domains/strawpoll.me.js                          | 2 +-
 plugins/domains/tumblr.com/tumblr.api.js                 | 2 +-
 plugins/domains/tumblr.com/tumblr.photo.js               | 2 +-
 plugins/domains/tumblr.com/tumblr.text.js                | 2 +-
 plugins/domains/tumblr.com/tumblr.video.js               | 2 +-
 plugins/domains/youtube.com/youtube.video.js             | 2 +-
 plugins/links/embedURL/ld-video.js                       | 2 +-
 plugins/links/hosted/23video-hosted.js                   | 2 +-
 plugins/links/oembed-photo-html.js                       | 2 +-
 plugins/links/oembed-video.js                            | 2 +-
 plugins/meta/ld-article.js                               | 2 +-
 plugins/meta/ld-product.js                               | 2 +-
 plugins/meta/oembed-description.js                       | 2 +-
 27 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/lib/html-utils.js b/lib/html-utils.js
index 0107ca9cd..1b3b559f3 100644
--- a/lib/html-utils.js
+++ b/lib/html-utils.js
@@ -1,4 +1,4 @@
-    import * as $ from 'cheerio';
+    import $ from 'cheerio';
     import * as _ from 'underscore';
     import CONFIG from '../config.js';
 
diff --git a/lib/plugins/system/cheerio.js b/lib/plugins/system/cheerio.js
index 3182f848e..f478adb5e 100644
--- a/lib/plugins/system/cheerio.js
+++ b/lib/plugins/system/cheerio.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 import * as htmlparser2 from "htmlparser2";
 var DomHandler = htmlparser2.DomHandler;
 
diff --git a/lib/plugins/system/oembed/oembed.js b/lib/plugins/system/oembed/oembed.js
index d3b868500..01faa0ed3 100644
--- a/lib/plugins/system/oembed/oembed.js
+++ b/lib/plugins/system/oembed/oembed.js
@@ -1,5 +1,5 @@
 import * as oembedUtils from './oembedUtils.js';
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 import * as entities from 'entities';
 import * as URL from 'url';
 import * as querystring from 'querystring';
diff --git a/lib/utils.js b/lib/utils.js
index 8331a3c03..34ce3bdfd 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -397,9 +397,7 @@ export function getCharset(string, doNotParse) {
     if (doNotParse) {
         charset = string.toUpperCase();
     } else if (string) {
-        console.log('-111', string)
         var m = string && string.match(/charset\s*=\s*([\w_-]+)/i);
-        console.log('-222')
         charset = m && m[1].toUpperCase();
     }
 
diff --git a/plugins/domains/art19.com.js b/plugins/domains/art19.com.js
index d9b440fb2..8a4834575 100644
--- a/plugins/domains/art19.com.js
+++ b/plugins/domains/art19.com.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/brightcove.com/players.brightcove.net.js b/plugins/domains/brightcove.com/players.brightcove.net.js
index 5d43b6de7..9c3eb6a5f 100644
--- a/plugins/domains/brightcove.com/players.brightcove.net.js
+++ b/plugins/domains/brightcove.com/players.brightcove.net.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 import * as utils from '../../../lib/utils.js';
 
 export default {
diff --git a/plugins/domains/cartodb.com.js b/plugins/domains/cartodb.com.js
index 3f2ac1184..83b104527 100644
--- a/plugins/domains/cartodb.com.js
+++ b/plugins/domains/cartodb.com.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 
 export default {
 
diff --git a/plugins/domains/codepen.io.js b/plugins/domains/codepen.io.js
index 9c8200624..0c8038642 100644
--- a/plugins/domains/codepen.io.js
+++ b/plugins/domains/codepen.io.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js
index 734fdda2b..cf7171d1b 100644
--- a/plugins/domains/instagram.com/instagram.com.js
+++ b/plugins/domains/instagram.com/instagram.com.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 import { decodeHTML5 as decodeHTML5 } from 'entities';
 
 export default {
diff --git a/plugins/domains/mixcloud.com.js b/plugins/domains/mixcloud.com.js
index 986327001..325b8eed3 100644
--- a/plugins/domains/mixcloud.com.js
+++ b/plugins/domains/mixcloud.com.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/scribd.com/scribd.com.js b/plugins/domains/scribd.com/scribd.com.js
index 33084c603..73b5a68cb 100644
--- a/plugins/domains/scribd.com/scribd.com.js
+++ b/plugins/domains/scribd.com/scribd.com.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import * as utils from '../../../lib/utils.js';
 import * as querystring from 'querystring';
 import * as URL from "url";
diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js
index 4961fe614..752631037 100644
--- a/plugins/domains/slideshare.net.js
+++ b/plugins/domains/slideshare.net.js
@@ -1,5 +1,5 @@
 import * as utils from '../../lib/utils.js';
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 
 export default {
 
diff --git a/plugins/domains/soundcloud.com/soundcloud.com.js b/plugins/domains/soundcloud.com/soundcloud.com.js
index 88f1196f6..40c0fdfea 100644
--- a/plugins/domains/soundcloud.com/soundcloud.com.js
+++ b/plugins/domains/soundcloud.com/soundcloud.com.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/speakerdeck.com.js b/plugins/domains/speakerdeck.com.js
index 53a2b417f..8d3791c9f 100644
--- a/plugins/domains/speakerdeck.com.js
+++ b/plugins/domains/speakerdeck.com.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 
 export default {
 
diff --git a/plugins/domains/strawpoll.me.js b/plugins/domains/strawpoll.me.js
index 9de15417b..507c189f2 100644
--- a/plugins/domains/strawpoll.me.js
+++ b/plugins/domains/strawpoll.me.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import * as entities from 'entities';
 
 export default {
diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js
index aeaa5e58b..268f96c18 100644
--- a/plugins/domains/tumblr.com/tumblr.api.js
+++ b/plugins/domains/tumblr.com/tumblr.api.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import * as _ from 'underscore';
 
 export default {
diff --git a/plugins/domains/tumblr.com/tumblr.photo.js b/plugins/domains/tumblr.com/tumblr.photo.js
index 2aa51a1e8..96a29129e 100644
--- a/plugins/domains/tumblr.com/tumblr.photo.js
+++ b/plugins/domains/tumblr.com/tumblr.photo.js
@@ -1,5 +1,5 @@
 import * as _ from 'underscore';
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 
 import tumblr_api from './tumblr.api.js';
 
diff --git a/plugins/domains/tumblr.com/tumblr.text.js b/plugins/domains/tumblr.com/tumblr.text.js
index 196c2e180..26c5a70e4 100644
--- a/plugins/domains/tumblr.com/tumblr.text.js
+++ b/plugins/domains/tumblr.com/tumblr.text.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import tumblr_api from './tumblr.api.js';
 
 export default {
diff --git a/plugins/domains/tumblr.com/tumblr.video.js b/plugins/domains/tumblr.com/tumblr.video.js
index a00d5ce01..0ced98eb9 100644
--- a/plugins/domains/tumblr.com/tumblr.video.js
+++ b/plugins/domains/tumblr.com/tumblr.video.js
@@ -1,4 +1,4 @@
-import * as $ from 'cheerio';
+import $ from 'cheerio';
 import tumblr_api from './tumblr.api.js';
 
 export default {
diff --git a/plugins/domains/youtube.com/youtube.video.js b/plugins/domains/youtube.com/youtube.video.js
index d9e9863ea..c8ee1c549 100644
--- a/plugins/domains/youtube.com/youtube.video.js
+++ b/plugins/domains/youtube.com/youtube.video.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 import * as querystring from 'querystring';
 import * as _ from 'underscore';
 import log from '../../../logging.js'
diff --git a/plugins/links/embedURL/ld-video.js b/plugins/links/embedURL/ld-video.js
index a076f50b7..f5defcaab 100644
--- a/plugins/links/embedURL/ld-video.js
+++ b/plugins/links/embedURL/ld-video.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 
 export default {
 
diff --git a/plugins/links/hosted/23video-hosted.js b/plugins/links/hosted/23video-hosted.js
index f5a160504..bebcb4d27 100644
--- a/plugins/links/hosted/23video-hosted.js
+++ b/plugins/links/hosted/23video-hosted.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 
 export default {
 
diff --git a/plugins/links/oembed-photo-html.js b/plugins/links/oembed-photo-html.js
index d3aee0865..5733b68d7 100644
--- a/plugins/links/oembed-photo-html.js
+++ b/plugins/links/oembed-photo-html.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 
 export default {
 
diff --git a/plugins/links/oembed-video.js b/plugins/links/oembed-video.js
index bacbe24b3..c983761b0 100644
--- a/plugins/links/oembed-video.js
+++ b/plugins/links/oembed-video.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 import * as entities from 'entities';
 
 export default {
diff --git a/plugins/meta/ld-article.js b/plugins/meta/ld-article.js
index 70441e90a..42ac59657 100644
--- a/plugins/meta/ld-article.js
+++ b/plugins/meta/ld-article.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 
 export default {
 
diff --git a/plugins/meta/ld-product.js b/plugins/meta/ld-product.js
index 5aa6b1d1e..595ffab93 100644
--- a/plugins/meta/ld-product.js
+++ b/plugins/meta/ld-product.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 
 export default {
 
diff --git a/plugins/meta/oembed-description.js b/plugins/meta/oembed-description.js
index 2f9b531df..8608c8b5f 100644
--- a/plugins/meta/oembed-description.js
+++ b/plugins/meta/oembed-description.js
@@ -1,4 +1,4 @@
-import * as cheerio from 'cheerio';
+import cheerio from 'cheerio';
 
 export default {
 

From 5e63e67c6794ae01e216470f765a87120e3cc1f9 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 18 Oct 2021 20:15:03 +0300
Subject: [PATCH 026/102] fetch: fix compression, use native

---
 lib/utils.js | 80 +++++++++-------------------------------------------
 package.json |  1 -
 2 files changed, 13 insertions(+), 68 deletions(-)

diff --git a/lib/utils.js b/lib/utils.js
index 34ce3bdfd..05cf714cb 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -3,9 +3,6 @@ import request from 'request';
 import fetch from 'node-fetch';
 import AbortController from 'abort-controller';
 import { Http2Agent } from './agent-http2.js';
-import * as zlib from 'zlib';
-import iltorb from 'iltorb';
-var decompressStream = iltorb;
 import * as iconv from 'iconv-lite';
 import * as async from 'async';
 import * as imagesize from 'probe-image-size';
@@ -210,84 +207,33 @@ export function getUrlFunctional(url, options, callbacks) {
                 headers: {
                     'User-Agent': options.user_agent || CONFIG.USER_AGENT,
                     'Connection': 'keep-alive',                    
-                    // TODO: temp disable gzip
-                    'Accept-Encoding': 'text/html',
-                    //'Accept-Encoding': 'gzip' + (sslProtocol ? ', br' : ''),
                     'Accept': '*/*'
+                    // gzip, br, etc - set by `node-fetch` `compress: true` option.
                 },
                 maxRedirects: options.maxRedirects || CONFIG.MAX_REDIRECTS,
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
                 followRedirect: options.followRedirect,
                 jar: jar,
-                signal: controller.signal
+
+                // New for `node-fetch`
+                signal: controller.signal,
+                compress: true
             }, options);
 
             fetch(request_options.url || request_options.uri, request_options)
                 .then(res => {
 
-                    var headers = prepareFetchHeaders(res.headers.raw());
-
-                    var contentEncoding = headers['content-encoding'];
-                    if (Array.isArray(contentEncoding)) {
-                        contentEncoding = contentEncoding[0];
+                    if (!options.asBuffer) {
+                        res.body.setEncoding("binary");
                     }
-                    contentEncoding = contentEncoding && contentEncoding.trim().toLowerCase();
-
-                    var zlibOptions = {
-                        flush: zlib.Z_SYNC_FLUSH,
-                        finishFlush: zlib.Z_SYNC_FLUSH
-                    };
-                    
-                    if (contentEncoding === 'br') {
-
-                        var br = decompressStream();
-
-                        // TODO:
-                        br.request = res.request;
-                        // TODO:
-                        br.status = res.status;
-                        br.headers = headers;
-
-                        if (!options.asBuffer) {
-                            br.setEncoding("binary");
-                        }
-
-                        // TODO:
-                        req.emit('response', br);
-                        res.body.pipe(br);
-
-                    } else if (contentEncoding === 'gzip' || contentEncoding === 'deflate') {
 
-                        var gunzip = contentEncoding === 'gzip' ? zlib.createGunzip(zlibOptions) : zlib.createInflate(zlibOptions);
+                    // TODO:
+                    var body = res.body;
+                    body.status = res.status;
+                    body.headers = prepareFetchHeaders(res.headers.raw());
 
-                        // TODO:
-                        gunzip.request = res.request;
-                        // TODO:
-                        gunzip.status = res.status;
-                        gunzip.headers = headers;
-
-                        if (!options.asBuffer) {
-                            gunzip.setEncoding("binary");
-                        }
-
-                        // TODO:
-                        req.emit('response', gunzip);
-                        res.body.pipe(gunzip);
-
-                    } else {
-
-                        if (!options.asBuffer) {
-                            // res.setEncoding("binary");
-                        }
-
-                        // TODO:
-                        var body = res.body;
-                        body.status = res.status;
-                        body.headers = headers;
-
-                        // TODO:
-                        req.emit('response', body);
-                    }
+                    // TODO:
+                    req.emit('response', body);
                 })
                 .catch(error => {
                     req.emit('error', error, {http2Agent: request_options.agent === http2Agent});
diff --git a/package.json b/package.json
index 4127c57fa..53ae6642c 100644
--- a/package.json
+++ b/package.json
@@ -32,7 +32,6 @@
         "htmlparser2": "3.9.2",
         "http-parser-js": "itteco/http-parser-js#magicode-fix-22",
         "iconv-lite": "0.4.17",
-        "iltorb": "^2.4.3",
         "jslint": "^0.12.1",
         "jsontoxml": "0.0.11",
         "memcached": "2.2.2",

From 24d5f84c2210b31a6f6f8fe30ab1e3d65bee730a Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 18 Oct 2021 23:40:48 +0300
Subject: [PATCH 027/102] node-fetch: use `redirect` and `follow` options

---
 lib/utils.js | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/lib/utils.js b/lib/utils.js
index 05cf714cb..cf6516a7f 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -198,8 +198,6 @@ export function getUrlFunctional(url, options, callbacks) {
     process.nextTick(function() {
         try {
 
-            var sslProtocol = /^(https:)?\/\//i.test(url);
-
             // TODO: review node-fetch options
             var request_options = prepareRequestOptions({
                 uri: url,
@@ -210,14 +208,14 @@ export function getUrlFunctional(url, options, callbacks) {
                     'Accept': '*/*'
                     // gzip, br, etc - set by `node-fetch` `compress: true` option.
                 },
-                maxRedirects: options.maxRedirects || CONFIG.MAX_REDIRECTS,
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-                followRedirect: options.followRedirect,
                 jar: jar,
 
                 // New for `node-fetch`
                 signal: controller.signal,
-                compress: true
+                compress: true,
+                redirect: options.followRedirect ? 'follow' : 'manual',
+                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
             }, options);
 
             fetch(request_options.url || request_options.uri, request_options)
@@ -306,15 +304,18 @@ var getHead = function(url, options) {
                     'User-Agent': CONFIG.USER_AGENT,
                     'Connection': 'close'
                 },
-                maxRedirects: options.maxRedirects || CONFIG.MAX_REDIRECTS,
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-                followRedirect: options.followRedirect,
                 jar: jar,
                 agentOptions: {
                     // getHead called for video, need check certificate.
                     rejectUnauthorized: true
                 },
-                signal: controller.signal
+
+                // New for `node-fetch`.
+                signal: controller.signal,
+                compress: true,
+                redirect: options.followRedirect ? 'follow' : 'manual',
+                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
             }, {
                 disableHttp2: options && options.disableHttp2
             });
@@ -1112,10 +1113,14 @@ var getUriStatusPrivate = function(uri, options, cb) {
         headers: {
             'User-Agent': CONFIG.USER_AGENT
         },
-        maxRedirects: CONFIG.MAX_REDIRECTS,
         timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
         jar: request.jar(),   //Enable cookies, uses new jar
-        signal: controller.signal
+
+        // New by 'node-fetch'.
+        signal: controller.signal,
+        compress: true,
+        redirect: 'follow', // Default value.
+        follow: CONFIG.MAX_REDIRECTS
     }, {
         disableHttp2: options.disableHttp2
     })

From c51068c6d3b9afb677a695b7cfc7844c987d0568 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 18:52:59 +0300
Subject: [PATCH 028/102] use fetch-h2

---
 lib/utils.js |  63 ++++----
 package.json |   3 +-
 yarn.lock    | 433 ++++++++-------------------------------------------
 3 files changed, 96 insertions(+), 403 deletions(-)

diff --git a/lib/utils.js b/lib/utils.js
index cf6516a7f..16549c889 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,8 +1,7 @@
 import * as events from 'events';
 import request from 'request';
-import fetch from 'node-fetch';
-import AbortController from 'abort-controller';
-import { Http2Agent } from './agent-http2.js';
+import { context, AbortController } from 'fetch-h2';
+const { fetch } = context( );
 import * as iconv from 'iconv-lite';
 import * as async from 'async';
 import * as imagesize from 'probe-image-size';
@@ -13,8 +12,6 @@ import { cache } from './cache.js';
 import * as htmlUtils from './html-utils.js';
 import log from '../logging.js';
 
-var http2Agent = new Http2Agent(/* options */);
-
 if (!global.CONFIG) {
     // TODO: review
      var config = await import('../config.js');
@@ -46,14 +43,6 @@ function maxTTL(ttl) {
     return ttl;
 }
 
-function prepareFetchHeaders(headers) {
-    var result = {};
-    for(var key in headers) {
-        result[key] = headers[key][0];
-    }
-    return result;
-}
-
 export function getMaxCacheTTLOverride(url, options) {
     var proxy = null;
     if (CONFIG.PROXY || (options && options.proxy)) {
@@ -195,6 +184,8 @@ export function getUrlFunctional(url, options, callbacks) {
         jar = request.jar();
     }
 
+    var response;
+
     process.nextTick(function() {
         try {
 
@@ -204,7 +195,7 @@ export function getUrlFunctional(url, options, callbacks) {
                 method: 'GET',
                 headers: {
                     'User-Agent': options.user_agent || CONFIG.USER_AGENT,
-                    'Connection': 'keep-alive',                    
+                    //'Connection': 'keep-alive',                    
                     'Accept': '*/*'
                     // gzip, br, etc - set by `node-fetch` `compress: true` option.
                 },
@@ -220,21 +211,22 @@ export function getUrlFunctional(url, options, callbacks) {
 
             fetch(request_options.url || request_options.uri, request_options)
                 .then(res => {
-
+                    response = res;
+                    return res.readable();
+                })
+                .then(stream => {
                     if (!options.asBuffer) {
-                        res.body.setEncoding("binary");
+                        stream.setEncoding("binary");
                     }
 
-                    // TODO:
-                    var body = res.body;
-                    body.status = res.status;
-                    body.headers = prepareFetchHeaders(res.headers.raw());
+                    stream.status = response.status;
+                    stream.headers = response.headers.toJSON();
 
                     // TODO:
-                    req.emit('response', body);
+                    req.emit('response', stream);
                 })
                 .catch(error => {
-                    req.emit('error', error, {http2Agent: request_options.agent === http2Agent});
+                    req.emit('error', error, {http2Agent: false/*request_options.agent === http2Agent*/});
                 });
 
 
@@ -270,12 +262,7 @@ export function getHeadFunctional(url, options, callbacks) {
             callbacks.onAbortController && callbacks.onAbortController(abortController);
         })
         .on('response', function(response) {
-            // TODO:
-            var body = response.body;
-            body.status = response.status;
-            body.headers = prepareFetchHeaders(response.headers.raw());
-
-            callbacks.onResponse && callbacks.onResponse(body);
+            callbacks.onResponse && callbacks.onResponse(response);
         });
 }
 
@@ -293,6 +280,8 @@ var getHead = function(url, options) {
         jar = request.jar();
     }
 
+    var response;
+
     process.nextTick(function() {
         try {
 
@@ -302,7 +291,7 @@ var getHead = function(url, options) {
                 method: 'HEAD',
                 headers: {
                     'User-Agent': CONFIG.USER_AGENT,
-                    'Connection': 'close'
+                    //'Connection': 'close'
                 },
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
                 jar: jar,
@@ -322,7 +311,15 @@ var getHead = function(url, options) {
 
             fetch(request_options.url || request_options.uri, request_options)
                 .then(res => {
-                    req.emit('response', res);
+                    response = res;
+                    return res.readable();
+                })
+                .then(stream => {
+
+                    stream.status = response.status;
+                    stream.headers = response.headers.toJSON();
+
+                    req.emit('response', stream);
                 })
                 .catch(error => {
                     req.emit('error', error);
@@ -1132,11 +1129,11 @@ var getUriStatusPrivate = function(uri, options, cb) {
 
                 var data = {
                     code: res.status,
-                    content_type: res.headers && res.headers['content-type'],
-                    content_length: res.headers && res.headers['content-length'] ? parseInt(res.headers['content-length'] || '0', 10) : null
+                    content_type: res.headers && res.headers.get('content-type'),
+                    content_length: res.headers && res.headers.get('content-length') ? parseInt(res.headers.get('content-length') || '0', 10) : null
                 };
                 if (options.checkHeaders) {
-                    data.headers = prepareFetchHeaders(res.headers.raw());
+                    data.headers = res.headers.toJSON();
                 }
                 cb(null, data);
             })
diff --git a/package.json b/package.json
index 53ae6642c..b37d52ce6 100644
--- a/package.json
+++ b/package.json
@@ -20,13 +20,13 @@
     },
     "license": "MIT",
     "dependencies": {
-        "abort-controller": "^3.0.0",
         "async": "2.4.1",
         "cheerio": "0.22.0",
         "chokidar": "^3.3.1",
         "ejs": "^3.1.6",
         "entities": "1.1.1",
         "express": "^4.16.3",
+        "fetch-h2": "^3.0.1",
         "globby": "^12.0.2",
         "graceful-cluster": "0.0.3",
         "htmlparser2": "3.9.2",
@@ -37,7 +37,6 @@
         "memcached": "2.2.2",
         "moment": "2.19.3",
         "node-cache": "1.*",
-        "node-fetch": "^3.0.0",
         "parse-iso-duration": "1.0.0",
         "probe-image-size": "^5.0.0",
         "readabilitySAX": "1.6.1",
diff --git a/yarn.lock b/yarn.lock
index 8ba060599..d02c71f1e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -43,18 +43,16 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-15.0.2.tgz#51e9c0920d1b45936ea04341aa3e2e58d339fb67"
   integrity sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==
 
+"@types/tough-cookie@^4.0.0":
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.1.tgz#8f80dd965ad81f3e1bc26d6f5c727e132721ff40"
+  integrity sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==
+
 abbrev@1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
   integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
 
-abort-controller@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392"
-  integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==
-  dependencies:
-    event-target-shim "^5.0.0"
-
 accepts@~1.3.7:
   version "1.3.7"
   resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd"
@@ -78,10 +76,10 @@ ajv@^6.12.3:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-ansi-regex@^2.0.0:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df"
-  integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8=
+already@^2.2.1:
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/already/-/already-2.2.1.tgz#98c257baa0d3fe62d85163ff288235ba11e3f188"
+  integrity sha512-qk6RIVMS/R1yTvBzfIL1T76PsIL7DIVCINoLuFw2YXKLpLtsTobqdChMs8m3OhuPS3CEE3+Ra5ibYiqdyogbsQ==
 
 ansi-styles@^3.2.1:
   version "3.2.1"
@@ -98,19 +96,6 @@ anymatch@~3.1.2:
     normalize-path "^3.0.0"
     picomatch "^2.0.4"
 
-aproba@^1.0.3:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
-  integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==
-
-are-we-there-yet@~1.1.2:
-  version "1.1.5"
-  resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21"
-  integrity sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==
-  dependencies:
-    delegates "^1.0.0"
-    readable-stream "^2.0.6"
-
 array-flatten@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -175,11 +160,6 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
-base64-js@^1.3.1:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
-  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
-
 bcrypt-pbkdf@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
@@ -200,15 +180,6 @@ bl@^2.2.1:
     readable-stream "^2.3.5"
     safe-buffer "^5.1.1"
 
-bl@^4.0.3:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
-  integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==
-  dependencies:
-    buffer "^5.5.0"
-    inherits "^2.0.4"
-    readable-stream "^3.4.0"
-
 bluebird@3.5.1:
   version "3.5.1"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9"
@@ -265,19 +236,16 @@ buffer-shims@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51"
   integrity sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=
 
-buffer@^5.5.0:
-  version "5.7.1"
-  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
-  integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
-  dependencies:
-    base64-js "^1.3.1"
-    ieee754 "^1.1.13"
-
 bytes@3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
   integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
 
+callguard@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/callguard/-/callguard-2.0.0.tgz#32f98348ff82cb1dfcf7d1b198108cf4f5b64c1f"
+  integrity sha512-I3nd+fuj20FK1qu00ImrbH+II+8ULS6ioYr9igqR1xyqySoqc3DiHEyUM0mkoAdKeLGg2CtGnO8R3VRQX5krpQ==
+
 caseless@~0.12.0:
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@@ -338,21 +306,11 @@ chokidar@^3.3.1:
   optionalDependencies:
     fsevents "~2.3.2"
 
-chownr@^1.1.1:
-  version "1.1.4"
-  resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
-  integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
-
 cluster-key-slot@^1.0.5:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
   integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
 
-code-point-at@^1.0.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
-  integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=
-
 color-convert@^1.9.0:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
@@ -402,11 +360,6 @@ connection-parse@0.0.x:
   resolved "https://registry.yarnpkg.com/connection-parse/-/connection-parse-0.0.7.tgz#18e7318aab06a699267372b10c5226d25a1c9a69"
   integrity sha1-GOcxiqsGppkmc3KxDFIm0locmmk=
 
-console-control-strings@^1.0.0, console-control-strings@~1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
-  integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=
-
 content-disposition@0.5.3:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.3.tgz#e130caf7e7279087c5616c2007d0485698984fbd"
@@ -461,11 +414,6 @@ dashdash@^1.12.0:
   dependencies:
     assert-plus "^1.0.0"
 
-data-uri-to-buffer@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz#594b8973938c5bc2c33046535785341abc4f3636"
-  integrity sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og==
-
 debug@2, debug@2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -487,13 +435,6 @@ debug@^3.1.0:
   dependencies:
     ms "^2.1.1"
 
-decompress-response@^4.2.0:
-  version "4.2.1"
-  resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986"
-  integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==
-  dependencies:
-    mimic-response "^2.0.0"
-
 deep-eql@^0.1.3:
   version "0.1.3"
   resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2"
@@ -501,11 +442,6 @@ deep-eql@^0.1.3:
   dependencies:
     type-detect "0.1.1"
 
-deep-extend@^0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
-  integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==
-
 deepmerge@^4.0.0:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955"
@@ -516,11 +452,6 @@ delayed-stream@~1.0.0:
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
   integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk=
 
-delegates@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
-  integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=
-
 denque@^1.1.0, denque@^1.4.1, denque@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de"
@@ -536,11 +467,6 @@ destroy@~1.0.4:
   resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80"
   integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=
 
-detect-libc@^1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
-  integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=
-
 diff@3.5.0:
   version "3.5.0"
   resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
@@ -625,13 +551,6 @@ encodeurl@~1.0.1, encodeurl@~1.0.2:
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
   integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=
 
-end-of-stream@^1.1.0, end-of-stream@^1.4.1:
-  version "1.4.4"
-  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
-  integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
-  dependencies:
-    once "^1.4.0"
-
 entities@0:
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-0.5.0.tgz#f611cb5ae221050e0012c66979503fd7ae19cc49"
@@ -657,21 +576,11 @@ etag@~1.8.1:
   resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
   integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
 
-event-target-shim@^5.0.0:
-  version "5.0.1"
-  resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789"
-  integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==
-
 exit@~0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c"
   integrity sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=
 
-expand-template@^2.0.3:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c"
-  integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==
-
 express@^4.16.3:
   version "4.17.1"
   resolved "https://registry.yarnpkg.com/express/-/express-4.17.1.tgz#4491fc38605cf51f8629d39c2b5d026f98a4c134"
@@ -760,12 +669,18 @@ feedparser@2.2.0:
     readable-stream "^2.2.2"
     sax "^1.2.1"
 
-fetch-blob@^3.1.2:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.1.2.tgz#6bc438675f3851ecea51758ac91f6a1cd1bacabd"
-  integrity sha512-hunJbvy/6OLjCD0uuhLdp0mMPzP/yd2ssd1t2FCJsaA7wkWhpbp9xfuNVpv7Ll4jFhzp6T4LAupSiV9uOeg0VQ==
+fetch-h2@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/fetch-h2/-/fetch-h2-3.0.1.tgz#e49ab81416ab90cf815c27584c6b073e584c928e"
+  integrity sha512-d7xOZHI4Wd31XJe9ib4MmncauFeojN5WOedOwDberYVysH0jcicu7WimZfsrEwzCV/EJPCSDwwwDTRWlCP8QtQ==
   dependencies:
-    web-streams-polyfill "^3.0.3"
+    "@types/tough-cookie" "^4.0.0"
+    already "^2.2.1"
+    callguard "^2.0.0"
+    get-stream "^6.0.1"
+    through2 "^4.0.2"
+    to-arraybuffer "^1.0.1"
+    tough-cookie "^4.0.0"
 
 filelist@^1.0.1:
   version "1.0.2"
@@ -823,11 +738,6 @@ fresh@0.5.2:
   resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7"
   integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=
 
-fs-constants@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
-  integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
-
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -838,19 +748,10 @@ fsevents@~2.3.2:
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
 
-gauge@~2.7.3:
-  version "2.7.4"
-  resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7"
-  integrity sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=
-  dependencies:
-    aproba "^1.0.3"
-    console-control-strings "^1.0.0"
-    has-unicode "^2.0.0"
-    object-assign "^4.1.0"
-    signal-exit "^3.0.0"
-    string-width "^1.0.1"
-    strip-ansi "^3.0.1"
-    wide-align "^1.1.0"
+get-stream@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7"
+  integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==
 
 getpass@^0.1.1:
   version "0.1.7"
@@ -859,11 +760,6 @@ getpass@^0.1.1:
   dependencies:
     assert-plus "^1.0.0"
 
-github-from-package@0.0.0:
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce"
-  integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=
-
 glob-parent@^5.1.2, glob-parent@~5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
@@ -935,11 +831,6 @@ has-flag@^3.0.0:
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd"
   integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0=
 
-has-unicode@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9"
-  integrity sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=
-
 hashring@3.2.x:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/hashring/-/hashring-3.2.0.tgz#fda4efde8aa22cdb97fb1d2a65e88401e1c144ce"
@@ -1032,27 +923,11 @@ iconv-lite@0.4.24:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
-ieee754@^1.1.13:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
-  integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
-
 ignore@^5.1.8:
   version "5.1.8"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
   integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==
 
-iltorb@^2.4.3:
-  version "2.4.5"
-  resolved "https://registry.yarnpkg.com/iltorb/-/iltorb-2.4.5.tgz#d64434b527099125c6839ed48b666247a172ef87"
-  integrity sha512-EMCMl3LnnNSZJS5QrxyZmMTaAC4+TJkM5woD+xbpm9RB+mFYCr7C05GFE3TEGCsVQSVHmjX+3sf5AiwsylNInQ==
-  dependencies:
-    detect-libc "^1.0.3"
-    nan "^2.14.0"
-    npmlog "^4.1.2"
-    prebuild-install "^5.3.3"
-    which-pm-runs "^1.0.0"
-
 inflight@^1.0.4:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
@@ -1061,7 +936,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3:
+inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -1071,11 +946,6 @@ inherits@2.0.3:
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
   integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
 
-ini@~1.3.0:
-  version "1.3.8"
-  resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c"
-  integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
-
 ipaddr.js@1.9.1:
   version "1.9.1"
   resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3"
@@ -1093,13 +963,6 @@ is-extglob@^2.1.1:
   resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
   integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=
 
-is-fullwidth-code-point@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb"
-  integrity sha1-754xOG8DGn8NZDr4L95QxFfvAMs=
-  dependencies:
-    number-is-nan "^1.0.0"
-
 is-glob@^4.0.1, is-glob@~4.0.1:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
@@ -1347,11 +1210,6 @@ mime@1.6.0, mime@^1.4.1:
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
 
-mimic-response@^2.0.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43"
-  integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==
-
 minimatch@3.0.4, minimatch@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
@@ -1364,21 +1222,11 @@ minimist@0.0.8:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
   integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=
 
-minimist@^1.2.0, minimist@^1.2.3:
-  version "1.2.5"
-  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
-  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
-
 minreq@0.2:
   version "0.2.3"
   resolved "https://registry.yarnpkg.com/minreq/-/minreq-0.2.3.tgz#9d7fc1d37f1cccccab55e402c12dbb78c7e3ab0d"
   integrity sha1-nX/B038czMyrVeQCwS27eMfjqw0=
 
-mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3:
-  version "0.5.3"
-  resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
-  integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
-
 mkdirp@0.5.1:
   version "0.5.1"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
@@ -1500,16 +1348,6 @@ multiparty@^4.1.2:
     safe-buffer "5.2.1"
     uid-safe "2.1.5"
 
-nan@^2.14.0:
-  version "2.14.2"
-  resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"
-  integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==
-
-napi-build-utils@^1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806"
-  integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==
-
 negotiator@0.6.2:
   version "0.6.2"
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
@@ -1520,13 +1358,6 @@ next-tick@^1.0.0:
   resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb"
   integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==
 
-node-abi@^2.7.0:
-  version "2.26.0"
-  resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.26.0.tgz#355d5d4bc603e856f74197adbf3f5117a396ba40"
-  integrity sha512-ag/Vos/mXXpWLLAYWsAoQdgS+gW7IwvgMLOgqopm/DbzAjazLltzgzpVMsFlgmo9TzG5hGXeaBZx2AI731RIsQ==
-  dependencies:
-    semver "^5.4.1"
-
 node-cache@1.*:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/node-cache/-/node-cache-1.1.0.tgz#186365032d2395bdff73404178fb2bc8981ace70"
@@ -1534,19 +1365,6 @@ node-cache@1.*:
   dependencies:
     underscore "*"
 
-node-fetch@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.0.0.tgz#79da7146a520036f2c5f644e4a26095f17e411ea"
-  integrity sha512-bKMI+C7/T/SPU1lKnbQbwxptpCrG9ashG+VkytmXCPZyuM9jB6VU+hY0oi4lC8LxTtAeWdckNCTa3nrGsAdA3Q==
-  dependencies:
-    data-uri-to-buffer "^3.0.1"
-    fetch-blob "^3.1.2"
-
-noop-logger@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2"
-  integrity sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=
-
 nopt@~4.0.1:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48"
@@ -1560,16 +1378,6 @@ normalize-path@^3.0.0, normalize-path@~3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
-npmlog@^4.0.1, npmlog@^4.1.2:
-  version "4.1.2"
-  resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b"
-  integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==
-  dependencies:
-    are-we-there-yet "~1.1.2"
-    console-control-strings "~1.1.0"
-    gauge "~2.7.3"
-    set-blocking "~2.0.0"
-
 nth-check@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c"
@@ -1577,21 +1385,11 @@ nth-check@~1.0.1:
   dependencies:
     boolbase "~1.0.0"
 
-number-is-nan@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
-  integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=
-
 oauth-sign@~0.9.0:
   version "0.9.0"
   resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
   integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==
 
-object-assign@^4.1.0:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
-  integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=
-
 on-finished@~2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947"
@@ -1599,7 +1397,7 @@ on-finished@~2.3.0:
   dependencies:
     ee-first "1.1.1"
 
-once@^1.3.0, once@^1.3.1, once@^1.4.0:
+once@^1.3.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
   integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
@@ -1671,27 +1469,6 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3:
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972"
   integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==
 
-prebuild-install@^5.3.3:
-  version "5.3.6"
-  resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-5.3.6.tgz#7c225568d864c71d89d07f8796042733a3f54291"
-  integrity sha512-s8Aai8++QQGi4sSbs/M1Qku62PFK49Jm1CbgXklGz4nmHveDq0wzJkg7Na5QbnO1uNH8K7iqx2EQ/mV0MZEmOg==
-  dependencies:
-    detect-libc "^1.0.3"
-    expand-template "^2.0.3"
-    github-from-package "0.0.0"
-    minimist "^1.2.3"
-    mkdirp-classic "^0.5.3"
-    napi-build-utils "^1.0.1"
-    node-abi "^2.7.0"
-    noop-logger "^0.1.1"
-    npmlog "^4.0.1"
-    pump "^3.0.0"
-    rc "^1.2.7"
-    simple-get "^3.0.3"
-    tar-fs "^2.0.0"
-    tunnel-agent "^0.6.0"
-    which-pm-runs "^1.0.0"
-
 probe-image-size@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-5.0.0.tgz#1b87d20340ab8fcdb4324ec77fbc8a5f53419878"
@@ -1721,19 +1498,11 @@ proxy-addr@~2.0.5:
     forwarded "~0.1.2"
     ipaddr.js "1.9.1"
 
-psl@^1.1.28:
+psl@^1.1.28, psl@^1.1.33:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
   integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
 
-pump@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
-  integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==
-  dependencies:
-    end-of-stream "^1.1.0"
-    once "^1.3.1"
-
 punycode@^2.1.0, punycode@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
@@ -1774,16 +1543,6 @@ raw-body@2.4.0:
     iconv-lite "0.4.24"
     unpipe "1.0.0"
 
-rc@^1.2.7:
-  version "1.2.8"
-  resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed"
-  integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==
-  dependencies:
-    deep-extend "^0.6.0"
-    ini "~1.3.0"
-    minimist "^1.2.0"
-    strip-json-comments "~2.0.1"
-
 readabilitySAX@1.6.1:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/readabilitySAX/-/readabilitySAX-1.6.1.tgz#4a040cdeeb52c62d7774da0e1b9708b72cebbd00"
@@ -1804,7 +1563,16 @@ readable-stream@1.0:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.2.2, readable-stream@^2.3.5:
+readable-stream@3:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
+  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
+readable-stream@^2.0.2, readable-stream@^2.2.2, readable-stream@^2.3.5:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
   integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==
@@ -1817,15 +1585,6 @@ readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.2.2, readable
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^3.1.1, readable-stream@^3.4.0:
-  version "3.6.0"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
-  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
-  dependencies:
-    inherits "^2.0.3"
-    string_decoder "^1.1.1"
-    util-deprecate "^1.0.1"
-
 readable-stream@~2.1.5:
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.1.5.tgz#66fa8b720e1438b364681f2ad1a63c618448c9d0"
@@ -1981,11 +1740,6 @@ sax@1.2.2, sax@^1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.2.tgz#fd8631a23bc7826bef5d871bdb87378c95647828"
   integrity sha1-/YYxojvHgmvvXYcb24c3jJVkeCg=
 
-semver@^5.4.1:
-  version "5.7.1"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
-  integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
-
 semver@^6.3.0:
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
@@ -2039,11 +1793,6 @@ serve-static@1.14.1:
     parseurl "~1.3.3"
     send "0.17.1"
 
-set-blocking@~2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
-  integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
-
 setprototypeof@1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656"
@@ -2064,25 +1813,6 @@ sift@13.5.2:
   resolved "https://registry.yarnpkg.com/sift/-/sift-13.5.2.tgz#24a715e13c617b086166cd04917d204a591c9da6"
   integrity sha512-+gxdEOMA2J+AI+fVsCqeNn7Tgx3M9ZN9jdi95939l1IJ8cZsqS8sqpJyOkic2SJk+1+98Uwryt/gL6XDaV+UZA==
 
-signal-exit@^3.0.0:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c"
-  integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==
-
-simple-concat@^1.0.0:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f"
-  integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==
-
-simple-get@^3.0.3:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-3.1.0.tgz#b45be062435e50d159540b576202ceec40b9c6b3"
-  integrity sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==
-  dependencies:
-    decompress-response "^4.2.0"
-    once "^1.3.1"
-    simple-concat "^1.0.0"
-
 simple-lru-cache@0.0.x:
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/simple-lru-cache/-/simple-lru-cache-0.0.2.tgz#d59cc3a193c1a5d0320f84ee732f6e4713e511dd"
@@ -2137,15 +1867,6 @@ stream-parser@~0.3.1:
   dependencies:
     debug "2"
 
-string-width@^1.0.1, "string-width@^1.0.2 || 2":
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3"
-  integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=
-  dependencies:
-    code-point-at "^1.0.0"
-    is-fullwidth-code-point "^1.0.0"
-    strip-ansi "^3.0.0"
-
 string_decoder@^1.1.1, string_decoder@~1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
@@ -2158,18 +1879,6 @@ string_decoder@~0.10.x:
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
   integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
 
-strip-ansi@^3.0.0, strip-ansi@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
-  integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=
-  dependencies:
-    ansi-regex "^2.0.0"
-
-strip-json-comments@~2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
-  integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo=
-
 superagent@^3.8.3:
   version "3.8.3"
   resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128"
@@ -2208,26 +1917,17 @@ supports-color@^5.3.0:
   dependencies:
     has-flag "^3.0.0"
 
-tar-fs@^2.0.0:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784"
-  integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==
+through2@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
+  integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
   dependencies:
-    chownr "^1.1.1"
-    mkdirp-classic "^0.5.2"
-    pump "^3.0.0"
-    tar-stream "^2.1.4"
+    readable-stream "3"
 
-tar-stream@^2.1.4:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
-  integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
-  dependencies:
-    bl "^4.0.3"
-    end-of-stream "^1.4.1"
-    fs-constants "^1.0.0"
-    inherits "^2.0.3"
-    readable-stream "^3.1.1"
+to-arraybuffer@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43"
+  integrity sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=
 
 to-regex-range@^5.0.1:
   version "5.0.1"
@@ -2241,6 +1941,15 @@ toidentifier@1.0.0:
   resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553"
   integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==
 
+tough-cookie@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4"
+  integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==
+  dependencies:
+    psl "^1.1.33"
+    punycode "^2.1.1"
+    universalify "^0.1.2"
+
 tough-cookie@~2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
@@ -2291,6 +2000,11 @@ underscore@*, underscore@^1.13.1, underscore@^1.8.3:
   resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.1.tgz#0c1c6bd2df54b6b69f2314066d65b6cde6fcf9d1"
   integrity sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==
 
+universalify@^0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
+  integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
+
 unpipe@1.0.0, unpipe@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
@@ -2332,23 +2046,6 @@ verror@1.10.0:
     core-util-is "1.0.2"
     extsprintf "^1.2.0"
 
-web-streams-polyfill@^3.0.3:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.1.1.tgz#1516f2d4ea8f1bdbfed15eb65cb2df87098c8364"
-  integrity sha512-Czi3fG883e96T4DLEPRvufrF2ydhOOW1+1a6c3gNjH2aIh50DNFBdfwh2AKoOf1rXvpvavAoA11Qdq9+BKjE0Q==
-
-which-pm-runs@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb"
-  integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=
-
-wide-align@^1.1.0:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457"
-  integrity sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==
-  dependencies:
-    string-width "^1.0.2 || 2"
-
 wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"

From a517bd80bb0f4466af97dfb2460d069a5ae118d2 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 19:01:48 +0300
Subject: [PATCH 029/102] cheerio: fix import htmlparser2

---
 lib/plugins/system/cheerio.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/plugins/system/cheerio.js b/lib/plugins/system/cheerio.js
index f478adb5e..d5832287e 100644
--- a/lib/plugins/system/cheerio.js
+++ b/lib/plugins/system/cheerio.js
@@ -1,5 +1,5 @@
 import cheerio from 'cheerio';
-import * as htmlparser2 from "htmlparser2";
+import htmlparser2 from "htmlparser2";
 var DomHandler = htmlparser2.DomHandler;
 
 export default {

From 35a0cc04f552266f1e644b7f62c46faea6b2cf3f Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 19:50:37 +0300
Subject: [PATCH 030/102] fix import imagesize

---
 lib/utils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/utils.js b/lib/utils.js
index 16549c889..83cde0483 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -4,7 +4,7 @@ import { context, AbortController } from 'fetch-h2';
 const { fetch } = context( );
 import * as iconv from 'iconv-lite';
 import * as async from 'async';
-import * as imagesize from 'probe-image-size';
+import imagesize from 'probe-image-size';
 import * as _ from 'underscore';
 import * as parseIsoDuration from 'parse-iso-duration';
 import * as entities from 'entities';

From 12a2b9d5deee7ca159c56fddebe69dbc22471d92 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 19:51:07 +0300
Subject: [PATCH 031/102] htmlparser: fix pause resume

---
 lib/plugins/system/htmlparser/htmlparser.js | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index ae20b1068..64ca1d874 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -132,16 +132,14 @@ export default {
                 // Init htmlparser handler.
                 var handler = new CollectingHandlerForMutliTarget();
                 handler.onNoHandlers = function() {
-                    // TODO:
-                    // if (!request.response.isPaused()) {
-                    //     request.response.pause();
-                    // }
+                    if (!resp.isPaused()) {
+                        resp.pause();
+                    }
                 };
                 handler.onFirstHandler = function() {
-                    // TODO:
-                    // if (request.response.isPaused()) {
-                    //     request.response.resume();
-                    // }
+                    if (resp.isPaused()) {
+                        resp.resume();
+                    }
                 };
                 var parser = new Parser(handler, {
                     lowerCaseTags: true

From 39d71050130623735bb14a3b287bd836a675c2fa Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 20:00:58 +0300
Subject: [PATCH 032/102] upgrade cheerio

---
 package.json |   2 +-
 yarn.lock    | 258 ++++++++++++++++++++++++++++-----------------------
 2 files changed, 141 insertions(+), 119 deletions(-)

diff --git a/package.json b/package.json
index b37d52ce6..c895daf96 100644
--- a/package.json
+++ b/package.json
@@ -21,7 +21,7 @@
     "license": "MIT",
     "dependencies": {
         "async": "2.4.1",
-        "cheerio": "0.22.0",
+        "cheerio": "^1.0.0-rc.10",
         "chokidar": "^3.3.1",
         "ejs": "^3.1.6",
         "entities": "1.1.1",
diff --git a/yarn.lock b/yarn.lock
index d02c71f1e..b11203644 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -201,7 +201,7 @@ body-parser@1.19.0, body-parser@^1.18.1:
     raw-body "2.4.0"
     type-is "~1.6.17"
 
-boolbase@~1.0.0:
+boolbase@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
   integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
@@ -269,27 +269,29 @@ chalk@^2.4.2:
     escape-string-regexp "^1.0.5"
     supports-color "^5.3.0"
 
-cheerio@0.22.0:
-  version "0.22.0"
-  resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-0.22.0.tgz#a9baa860a3f9b595a6b81b1a86873121ed3a269e"
-  integrity sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=
-  dependencies:
-    css-select "~1.2.0"
-    dom-serializer "~0.1.0"
-    entities "~1.1.1"
-    htmlparser2 "^3.9.1"
-    lodash.assignin "^4.0.9"
-    lodash.bind "^4.1.4"
-    lodash.defaults "^4.0.1"
-    lodash.filter "^4.4.0"
-    lodash.flatten "^4.2.0"
-    lodash.foreach "^4.3.0"
-    lodash.map "^4.4.0"
-    lodash.merge "^4.4.0"
-    lodash.pick "^4.2.1"
-    lodash.reduce "^4.4.0"
-    lodash.reject "^4.4.0"
-    lodash.some "^4.4.0"
+cheerio-select@^1.5.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/cheerio-select/-/cheerio-select-1.5.0.tgz#faf3daeb31b17c5e1a9dabcee288aaf8aafa5823"
+  integrity sha512-qocaHPv5ypefh6YNxvnbABM07KMxExbtbfuJoIie3iZXX1ERwYmJcIiRrr9H05ucQP1k28dav8rpdDgjQd8drg==
+  dependencies:
+    css-select "^4.1.3"
+    css-what "^5.0.1"
+    domelementtype "^2.2.0"
+    domhandler "^4.2.0"
+    domutils "^2.7.0"
+
+cheerio@^1.0.0-rc.10:
+  version "1.0.0-rc.10"
+  resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.10.tgz#2ba3dcdfcc26e7956fc1f440e61d51c643379f3e"
+  integrity sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==
+  dependencies:
+    cheerio-select "^1.5.0"
+    dom-serializer "^1.3.2"
+    domhandler "^4.2.0"
+    htmlparser2 "^6.1.0"
+    parse5 "^6.0.1"
+    parse5-htmlparser2-tree-adapter "^6.0.1"
+    tslib "^2.2.0"
 
 chokidar@^3.3.1:
   version "3.5.2"
@@ -387,25 +389,31 @@ cookiejar@^2.1.0:
   resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.2.tgz#dd8a235530752f988f9a0844f3fc589e3111125c"
   integrity sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==
 
-core-util-is@1.0.2, core-util-is@~1.0.0:
+core-util-is@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
-css-select@~1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
-  integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=
+core-util-is@~1.0.0:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"
+  integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==
+
+css-select@^4.1.3:
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.1.3.tgz#a70440f70317f2669118ad74ff105e65849c7067"
+  integrity sha512-gT3wBNd9Nj49rAbmtFHj1cljIAOLYSX1nZ8CB7TBO3INYckygm5B7LISU/szY//YmdiSLbJvDLOx9VnMVpMBxA==
   dependencies:
-    boolbase "~1.0.0"
-    css-what "2.1"
-    domutils "1.5.1"
-    nth-check "~1.0.1"
+    boolbase "^1.0.0"
+    css-what "^5.0.0"
+    domhandler "^4.2.0"
+    domutils "^2.6.0"
+    nth-check "^2.0.0"
 
-css-what@2.1:
-  version "2.1.3"
-  resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.3.tgz#a6d7604573365fe74686c3f311c56513d88285f2"
-  integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==
+css-what@^5.0.0, css-what@^5.0.1:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/css-what/-/css-what-5.1.0.tgz#3f7b707aadf633baf62c2ceb8579b545bb40f7fe"
+  integrity sha512-arSMRWIIFY0hV8pIxZMEfmMI47Wj3R/aWpZDDxWYCPEiOMv6tfOrnpDtgxBYPEQD4V0Y/958+1TdC3iWTFcUPw==
 
 dashdash@^1.12.0:
   version "1.14.1"
@@ -479,19 +487,33 @@ dir-glob@^3.0.1:
   dependencies:
     path-type "^4.0.0"
 
-dom-serializer@0, dom-serializer@~0.1.0:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
-  integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==
+dom-serializer@0:
+  version "0.2.2"
+  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51"
+  integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==
   dependencies:
-    domelementtype "^1.3.0"
-    entities "^1.1.1"
+    domelementtype "^2.0.1"
+    entities "^2.0.0"
+
+dom-serializer@^1.0.1, dom-serializer@^1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.3.2.tgz#6206437d32ceefaec7161803230c7a20bc1b4d91"
+  integrity sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig==
+  dependencies:
+    domelementtype "^2.0.1"
+    domhandler "^4.2.0"
+    entities "^2.0.0"
 
 domelementtype@1, domelementtype@^1.3.0:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
   integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
 
+domelementtype@^2.0.1, domelementtype@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.2.0.tgz#9a0b6c2782ed6a1c7323d42267183df9bd8b1d57"
+  integrity sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==
+
 domhandler@2.0:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.0.3.tgz#889f8df626403af0788e29d66d5d5c6f7ebf0fd6"
@@ -506,6 +528,13 @@ domhandler@^2.3.0:
   dependencies:
     domelementtype "1"
 
+domhandler@^4.0.0, domhandler@^4.2.0:
+  version "4.2.2"
+  resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.2.tgz#e825d721d19a86b8c201a35264e226c678ee755f"
+  integrity sha512-PzE9aBMsdZO8TK4BnuJwH0QT41wgMbRzuZrHUcpYncEjmQazq8QEaBWgLG7ZyC/DAZKEgglpIA6j4Qn/HmxS3w==
+  dependencies:
+    domelementtype "^2.2.0"
+
 domutils@1.1:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.1.6.tgz#bddc3de099b9a2efacc51c623f28f416ecc57485"
@@ -513,14 +542,23 @@ domutils@1.1:
   dependencies:
     domelementtype "1"
 
-domutils@1.5.1, domutils@^1.5.1:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf"
-  integrity sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=
+domutils@^1.5.1:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a"
+  integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==
   dependencies:
     dom-serializer "0"
     domelementtype "1"
 
+domutils@^2.5.2, domutils@^2.6.0, domutils@^2.7.0:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135"
+  integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==
+  dependencies:
+    dom-serializer "^1.0.1"
+    domelementtype "^2.2.0"
+    domhandler "^4.2.0"
+
 double-ended-queue@^2.1.0-0:
   version "2.1.0-0"
   resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c"
@@ -556,11 +594,21 @@ entities@0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-0.5.0.tgz#f611cb5ae221050e0012c66979503fd7ae19cc49"
   integrity sha1-9hHLWuIhBQ4AEsZpeVA/164ZzEk=
 
-entities@1.1.1, entities@^1.1.1, entities@~1.1.1:
+entities@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0"
   integrity sha1-blwtClYhtdra7O+AuQ7ftc13cvA=
 
+entities@^1.1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
+  integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
+
+entities@^2.0.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
+  integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
+
 escape-html@~1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
@@ -854,7 +902,7 @@ htmlparser2@3.0:
     domutils "1.1"
     readable-stream "1.0"
 
-htmlparser2@3.9.2, htmlparser2@^3.9.1:
+htmlparser2@3.9.2:
   version "3.9.2"
   resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338"
   integrity sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=
@@ -866,6 +914,16 @@ htmlparser2@3.9.2, htmlparser2@^3.9.1:
     inherits "^2.0.1"
     readable-stream "^2.0.2"
 
+htmlparser2@^6.1.0:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
+  integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==
+  dependencies:
+    domelementtype "^2.0.1"
+    domhandler "^4.0.0"
+    domutils "^2.5.2"
+    entities "^2.0.0"
+
 http-errors@1.7.2, http-errors@~1.7.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f"
@@ -1067,36 +1125,6 @@ lodash.assign@^4.2.0:
   resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
   integrity sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=
 
-lodash.assignin@^4.0.9:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/lodash.assignin/-/lodash.assignin-4.2.0.tgz#ba8df5fb841eb0a3e8044232b0e263a8dc6a28a2"
-  integrity sha1-uo31+4QesKPoBEIysOJjqNxqKKI=
-
-lodash.bind@^4.1.4:
-  version "4.2.1"
-  resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35"
-  integrity sha1-euMBfpOWIqwxt9fX3LGzTbFpDTU=
-
-lodash.defaults@^4.0.1:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
-  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
-
-lodash.filter@^4.4.0:
-  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
-  integrity sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4=
-
-lodash.flatten@^4.2.0:
-  version "4.4.0"
-  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
-  integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
-
-lodash.foreach@^4.3.0:
-  version "4.5.0"
-  resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
-  integrity sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=
-
 lodash.get@^4.4.2:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
@@ -1107,36 +1135,6 @@ lodash.has@^4.5.2:
   resolved "https://registry.yarnpkg.com/lodash.has/-/lodash.has-4.5.2.tgz#d19f4dc1095058cccbe2b0cdf4ee0fe4aa37c862"
   integrity sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=
 
-lodash.map@^4.4.0:
-  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3"
-  integrity sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=
-
-lodash.merge@^4.4.0:
-  version "4.6.2"
-  resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
-  integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==
-
-lodash.pick@^4.2.1:
-  version "4.4.0"
-  resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
-  integrity sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=
-
-lodash.reduce@^4.4.0:
-  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/lodash.reduce/-/lodash.reduce-4.6.0.tgz#f1ab6b839299ad48f784abbf476596f03b914d3b"
-  integrity sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs=
-
-lodash.reject@^4.4.0:
-  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/lodash.reject/-/lodash.reject-4.6.0.tgz#80d6492dc1470864bbf583533b651f42a9f52415"
-  integrity sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU=
-
-lodash.some@^4.4.0:
-  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d"
-  integrity sha1-G7nzFO9ri63tE7VJFpsqlF62jk0=
-
 lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
@@ -1378,12 +1376,12 @@ normalize-path@^3.0.0, normalize-path@~3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
-nth-check@~1.0.1:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c"
-  integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==
+nth-check@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.1.tgz#2efe162f5c3da06a28959fbd3db75dbeea9f0fc2"
+  integrity sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==
   dependencies:
-    boolbase "~1.0.0"
+    boolbase "^1.0.0"
 
 oauth-sign@~0.9.0:
   version "0.9.0"
@@ -1439,6 +1437,18 @@ parse-iso-duration@1.0.0:
   resolved "https://registry.yarnpkg.com/parse-iso-duration/-/parse-iso-duration-1.0.0.tgz#b923ab898a8ff8f42bdc9ee5db6e22808c48a864"
   integrity sha1-uSOriYqP+PQr3J7l224igIxIqGQ=
 
+parse5-htmlparser2-tree-adapter@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz#2cdf9ad823321140370d4dbf5d3e92c7c8ddc6e6"
+  integrity sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==
+  dependencies:
+    parse5 "^6.0.1"
+
+parse5@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b"
+  integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==
+
 parseurl@~1.3.3:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
@@ -1718,7 +1728,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
-safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2:
+safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@@ -1867,18 +1877,25 @@ stream-parser@~0.3.1:
   dependencies:
     debug "2"
 
-string_decoder@^1.1.1, string_decoder@~1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
-  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+string_decoder@^1.1.1:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
+  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
   dependencies:
-    safe-buffer "~5.1.0"
+    safe-buffer "~5.2.0"
 
 string_decoder@~0.10.x:
   version "0.10.31"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
   integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
 
+string_decoder@~1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
+  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+  dependencies:
+    safe-buffer "~5.1.0"
+
 superagent@^3.8.3:
   version "3.8.3"
   resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128"
@@ -1958,6 +1975,11 @@ tough-cookie@~2.5.0:
     psl "^1.1.28"
     punycode "^2.1.1"
 
+tslib@^2.2.0:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
+  integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==
+
 tunnel-agent@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"

From bce7fbde00d45c43353f7048f6e3616907bd464e Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 22:14:38 +0300
Subject: [PATCH 033/102] upgrade `htmlparser2`

---
 .../CollectingHandlerForMutliTarget.js        | 68 +++++++------------
 lib/plugins/system/htmlparser/htmlparser.js   |  3 +-
 package.json                                  |  3 +-
 yarn.lock                                     | 65 ++++++------------
 4 files changed, 48 insertions(+), 91 deletions(-)

diff --git a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
index 6bf05a7fc..efe915b80 100644
--- a/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
+++ b/lib/plugins/system/htmlparser/CollectingHandlerForMutliTarget.js
@@ -9,35 +9,30 @@ export function CollectingHandlerForMutliTarget(cbsArray){
     this._cbsArray = cbsArray || [];
     this.events = [];
 }
-import htmlparser2 from "htmlparser2";
-const EVENTS = htmlparser2.EVENTS;
 
-Object.keys(EVENTS).forEach(function(name) {
-
-    if (EVENTS[name] === 0) {
-
-        name = "on" + name;
-        CollectingHandlerForMutliTarget.prototype[name] = function() {
-            this.emitCb([name]);
-        };
-
-    } else if (EVENTS[name] === 1) {
-
-        name = "on" + name;
-        CollectingHandlerForMutliTarget.prototype[name] = function(a) {
-            this.emitCb([name, a]);
-        };
-
-    } else if (EVENTS[name] === 2) {
-
-        name = "on" + name;
-        CollectingHandlerForMutliTarget.prototype[name] = function(a, b) {
-            this.emitCb([name, a, b]);
-        };
-
-    } else {
-        throw Error("wrong number of arguments");
-    }
+// Got from `export interface Handler {` (Parser.d.ts).
+const EVENTS = [
+    'onparserinit',
+    'onreset',
+    'onend',
+    'onerror',
+    'onclosetag',
+    'onopentagname',
+    'onattribute',
+    'onopentag',
+    'ontext',
+    'oncomment',
+    'oncdatastart',
+    'oncdataend',
+    'oncommentend',
+    'onprocessinginstruction',
+];
+
+EVENTS.forEach(function(name) {
+    CollectingHandlerForMutliTarget.prototype[name] = function() {
+        let args = [name, ...arguments];
+        this.emitCb(args);
+    };
 });
 
 CollectingHandlerForMutliTarget.prototype.addHandler = function(cbs) {
@@ -94,20 +89,9 @@ CollectingHandlerForMutliTarget.prototype._onreset = function(cbs) {
 CollectingHandlerForMutliTarget.prototype.callCb = function(event, cbs) {
 
     function cb(cbs) {
-
-        if (cbs[event[0]]) {
-
-            var num = event.length;
-
-            if (num === 1) {
-                cbs[event[0]]();
-
-            } else if (num === 2) {
-                cbs[event[0]](event[1]);
-
-            } else {
-                cbs[event[0]](event[1], event[2]);
-            }
+        const name = event[0];
+        if (cbs[name]) {
+            cbs[name].apply(cbs, event.slice(1));
         }
     }
 
diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 64ca1d874..4a11dbf60 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -1,7 +1,6 @@
 import * as _ from 'underscore';
 import * as urlLib from 'url';
-import * as htmlparser2 from 'htmlparser2';
-var Parser = htmlparser2.Parser;
+import { Parser } from 'htmlparser2';
 import { cache } from '../../../cache.js';
 import * as utils from '../../../utils.js';
 import * as libUtils from '../../../utils.js';
diff --git a/package.json b/package.json
index c895daf96..651616a08 100644
--- a/package.json
+++ b/package.json
@@ -29,8 +29,7 @@
         "fetch-h2": "^3.0.1",
         "globby": "^12.0.2",
         "graceful-cluster": "0.0.3",
-        "htmlparser2": "3.9.2",
-        "http-parser-js": "itteco/http-parser-js#magicode-fix-22",
+        "htmlparser2": "^7.1.2",
         "iconv-lite": "0.4.17",
         "jslint": "^0.12.1",
         "jsontoxml": "0.0.11",
diff --git a/yarn.lock b/yarn.lock
index b11203644..15d64c6f7 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -487,14 +487,6 @@ dir-glob@^3.0.1:
   dependencies:
     path-type "^4.0.0"
 
-dom-serializer@0:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51"
-  integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==
-  dependencies:
-    domelementtype "^2.0.1"
-    entities "^2.0.0"
-
 dom-serializer@^1.0.1, dom-serializer@^1.3.2:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.3.2.tgz#6206437d32ceefaec7161803230c7a20bc1b4d91"
@@ -504,7 +496,7 @@ dom-serializer@^1.0.1, dom-serializer@^1.3.2:
     domhandler "^4.2.0"
     entities "^2.0.0"
 
-domelementtype@1, domelementtype@^1.3.0:
+domelementtype@1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
   integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
@@ -521,14 +513,7 @@ domhandler@2.0:
   dependencies:
     domelementtype "1"
 
-domhandler@^2.3.0:
-  version "2.4.2"
-  resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803"
-  integrity sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==
-  dependencies:
-    domelementtype "1"
-
-domhandler@^4.0.0, domhandler@^4.2.0:
+domhandler@^4.0.0, domhandler@^4.2.0, domhandler@^4.2.2:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.2.tgz#e825d721d19a86b8c201a35264e226c678ee755f"
   integrity sha512-PzE9aBMsdZO8TK4BnuJwH0QT41wgMbRzuZrHUcpYncEjmQazq8QEaBWgLG7ZyC/DAZKEgglpIA6j4Qn/HmxS3w==
@@ -542,15 +527,7 @@ domutils@1.1:
   dependencies:
     domelementtype "1"
 
-domutils@^1.5.1:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a"
-  integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==
-  dependencies:
-    dom-serializer "0"
-    domelementtype "1"
-
-domutils@^2.5.2, domutils@^2.6.0, domutils@^2.7.0:
+domutils@^2.5.2, domutils@^2.6.0, domutils@^2.7.0, domutils@^2.8.0:
   version "2.8.0"
   resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135"
   integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==
@@ -599,16 +576,16 @@ entities@1.1.1:
   resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0"
   integrity sha1-blwtClYhtdra7O+AuQ7ftc13cvA=
 
-entities@^1.1.1:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
-  integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
-
 entities@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
   integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==
 
+entities@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-3.0.1.tgz#2b887ca62585e96db3903482d336c1006c3001d4"
+  integrity sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q==
+
 escape-html@~1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
@@ -902,18 +879,6 @@ htmlparser2@3.0:
     domutils "1.1"
     readable-stream "1.0"
 
-htmlparser2@3.9.2:
-  version "3.9.2"
-  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338"
-  integrity sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=
-  dependencies:
-    domelementtype "^1.3.0"
-    domhandler "^2.3.0"
-    domutils "^1.5.1"
-    entities "^1.1.1"
-    inherits "^2.0.1"
-    readable-stream "^2.0.2"
-
 htmlparser2@^6.1.0:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
@@ -924,6 +889,16 @@ htmlparser2@^6.1.0:
     domutils "^2.5.2"
     entities "^2.0.0"
 
+htmlparser2@^7.1.2:
+  version "7.1.2"
+  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-7.1.2.tgz#587923d38f03bc89e03076e00cba2c7473f37f7c"
+  integrity sha512-d6cqsbJba2nRdg8WW2okyD4ceonFHn9jLFxhwlNcLhQWcFPdxXeJulgOLjLKtAK9T6ahd+GQNZwG9fjmGW7lyg==
+  dependencies:
+    domelementtype "^2.0.1"
+    domhandler "^4.2.2"
+    domutils "^2.8.0"
+    entities "^3.0.1"
+
 http-errors@1.7.2, http-errors@~1.7.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f"
@@ -994,7 +969,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3:
+inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -1582,7 +1557,7 @@ readable-stream@3:
     string_decoder "^1.1.1"
     util-deprecate "^1.0.1"
 
-readable-stream@^2.0.2, readable-stream@^2.2.2, readable-stream@^2.3.5:
+readable-stream@^2.2.2, readable-stream@^2.3.5:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
   integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==

From 1b5c4f14a0a7912f4fac42f617db71d3572c3a86 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 22:26:38 +0300
Subject: [PATCH 034/102] fix `cheerio` import

---
 lib/html-utils.js                                        | 3 ++-
 lib/plugins/system/cheerio.js                            | 3 ++-
 lib/plugins/system/oembed/oembed.js                      | 3 ++-
 plugins/domains/art19.com.js                             | 3 ++-
 plugins/domains/brightcove.com/players.brightcove.net.js | 3 ++-
 plugins/domains/cartodb.com.js                           | 3 ++-
 plugins/domains/codepen.io.js                            | 3 ++-
 plugins/domains/instagram.com/instagram.com.js           | 3 ++-
 plugins/domains/mixcloud.com.js                          | 3 ++-
 plugins/domains/scribd.com/scribd.com.js                 | 3 ++-
 plugins/domains/slideshare.net.js                        | 3 ++-
 plugins/domains/soundcloud.com/soundcloud.com.js         | 3 ++-
 plugins/domains/speakerdeck.com.js                       | 3 ++-
 plugins/domains/strawpoll.me.js                          | 3 ++-
 plugins/domains/tumblr.com/tumblr.api.js                 | 3 ++-
 plugins/domains/tumblr.com/tumblr.photo.js               | 3 ++-
 plugins/domains/tumblr.com/tumblr.text.js                | 3 ++-
 plugins/domains/tumblr.com/tumblr.video.js               | 3 ++-
 plugins/domains/youtube.com/youtube.video.js             | 3 ++-
 plugins/links/embedURL/ld-video.js                       | 3 ++-
 plugins/links/hosted/23video-hosted.js                   | 3 ++-
 plugins/links/oembed-photo-html.js                       | 3 ++-
 plugins/links/oembed-video.js                            | 3 ++-
 plugins/meta/ld-article.js                               | 3 ++-
 plugins/meta/ld-product.js                               | 3 ++-
 plugins/meta/oembed-description.js                       | 3 ++-
 26 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/lib/html-utils.js b/lib/html-utils.js
index 1b3b559f3..00d93132c 100644
--- a/lib/html-utils.js
+++ b/lib/html-utils.js
@@ -1,4 +1,5 @@
-    import $ from 'cheerio';
+    import cheerio_pkg from 'cheerio';
+    const $ = cheerio_pkg.default;
     import * as _ from 'underscore';
     import CONFIG from '../config.js';
 
diff --git a/lib/plugins/system/cheerio.js b/lib/plugins/system/cheerio.js
index d5832287e..9973f2ea4 100644
--- a/lib/plugins/system/cheerio.js
+++ b/lib/plugins/system/cheerio.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 import htmlparser2 from "htmlparser2";
 var DomHandler = htmlparser2.DomHandler;
 
diff --git a/lib/plugins/system/oembed/oembed.js b/lib/plugins/system/oembed/oembed.js
index 01faa0ed3..d7d7ea4ef 100644
--- a/lib/plugins/system/oembed/oembed.js
+++ b/lib/plugins/system/oembed/oembed.js
@@ -1,5 +1,6 @@
 import * as oembedUtils from './oembedUtils.js';
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 import * as entities from 'entities';
 import * as URL from 'url';
 import * as querystring from 'querystring';
diff --git a/plugins/domains/art19.com.js b/plugins/domains/art19.com.js
index 8a4834575..69ee1274d 100644
--- a/plugins/domains/art19.com.js
+++ b/plugins/domains/art19.com.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/brightcove.com/players.brightcove.net.js b/plugins/domains/brightcove.com/players.brightcove.net.js
index 9c3eb6a5f..6136dbf8c 100644
--- a/plugins/domains/brightcove.com/players.brightcove.net.js
+++ b/plugins/domains/brightcove.com/players.brightcove.net.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 import * as utils from '../../../lib/utils.js';
 
 export default {
diff --git a/plugins/domains/cartodb.com.js b/plugins/domains/cartodb.com.js
index 83b104527..2d172e72e 100644
--- a/plugins/domains/cartodb.com.js
+++ b/plugins/domains/cartodb.com.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/domains/codepen.io.js b/plugins/domains/codepen.io.js
index 0c8038642..d13b2f7d3 100644
--- a/plugins/domains/codepen.io.js
+++ b/plugins/domains/codepen.io.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js
index cf7171d1b..9e4ef51ab 100644
--- a/plugins/domains/instagram.com/instagram.com.js
+++ b/plugins/domains/instagram.com/instagram.com.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 import { decodeHTML5 as decodeHTML5 } from 'entities';
 
 export default {
diff --git a/plugins/domains/mixcloud.com.js b/plugins/domains/mixcloud.com.js
index 325b8eed3..3774907a2 100644
--- a/plugins/domains/mixcloud.com.js
+++ b/plugins/domains/mixcloud.com.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/scribd.com/scribd.com.js b/plugins/domains/scribd.com/scribd.com.js
index 73b5a68cb..b4d3504a6 100644
--- a/plugins/domains/scribd.com/scribd.com.js
+++ b/plugins/domains/scribd.com/scribd.com.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import * as utils from '../../../lib/utils.js';
 import * as querystring from 'querystring';
 import * as URL from "url";
diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js
index 752631037..92bc96b00 100644
--- a/plugins/domains/slideshare.net.js
+++ b/plugins/domains/slideshare.net.js
@@ -1,5 +1,6 @@
 import * as utils from '../../lib/utils.js';
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/domains/soundcloud.com/soundcloud.com.js b/plugins/domains/soundcloud.com/soundcloud.com.js
index 40c0fdfea..536c6b3f3 100644
--- a/plugins/domains/soundcloud.com/soundcloud.com.js
+++ b/plugins/domains/soundcloud.com/soundcloud.com.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import * as querystring from 'querystring';
 import * as URL from "url";
 
diff --git a/plugins/domains/speakerdeck.com.js b/plugins/domains/speakerdeck.com.js
index 8d3791c9f..8cd6a3c15 100644
--- a/plugins/domains/speakerdeck.com.js
+++ b/plugins/domains/speakerdeck.com.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/domains/strawpoll.me.js b/plugins/domains/strawpoll.me.js
index 507c189f2..f3fff1a36 100644
--- a/plugins/domains/strawpoll.me.js
+++ b/plugins/domains/strawpoll.me.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import * as entities from 'entities';
 
 export default {
diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js
index 268f96c18..079399b75 100644
--- a/plugins/domains/tumblr.com/tumblr.api.js
+++ b/plugins/domains/tumblr.com/tumblr.api.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import * as _ from 'underscore';
 
 export default {
diff --git a/plugins/domains/tumblr.com/tumblr.photo.js b/plugins/domains/tumblr.com/tumblr.photo.js
index 96a29129e..1b73ad6fc 100644
--- a/plugins/domains/tumblr.com/tumblr.photo.js
+++ b/plugins/domains/tumblr.com/tumblr.photo.js
@@ -1,5 +1,6 @@
 import * as _ from 'underscore';
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 
 import tumblr_api from './tumblr.api.js';
 
diff --git a/plugins/domains/tumblr.com/tumblr.text.js b/plugins/domains/tumblr.com/tumblr.text.js
index 26c5a70e4..b857edbae 100644
--- a/plugins/domains/tumblr.com/tumblr.text.js
+++ b/plugins/domains/tumblr.com/tumblr.text.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import tumblr_api from './tumblr.api.js';
 
 export default {
diff --git a/plugins/domains/tumblr.com/tumblr.video.js b/plugins/domains/tumblr.com/tumblr.video.js
index 0ced98eb9..2a864ff58 100644
--- a/plugins/domains/tumblr.com/tumblr.video.js
+++ b/plugins/domains/tumblr.com/tumblr.video.js
@@ -1,4 +1,5 @@
-import $ from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 import tumblr_api from './tumblr.api.js';
 
 export default {
diff --git a/plugins/domains/youtube.com/youtube.video.js b/plugins/domains/youtube.com/youtube.video.js
index c8ee1c549..dc9e1d370 100644
--- a/plugins/domains/youtube.com/youtube.video.js
+++ b/plugins/domains/youtube.com/youtube.video.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 import * as querystring from 'querystring';
 import * as _ from 'underscore';
 import log from '../../../logging.js'
diff --git a/plugins/links/embedURL/ld-video.js b/plugins/links/embedURL/ld-video.js
index f5defcaab..847185a2f 100644
--- a/plugins/links/embedURL/ld-video.js
+++ b/plugins/links/embedURL/ld-video.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/links/hosted/23video-hosted.js b/plugins/links/hosted/23video-hosted.js
index bebcb4d27..f07fda3a1 100644
--- a/plugins/links/hosted/23video-hosted.js
+++ b/plugins/links/hosted/23video-hosted.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/links/oembed-photo-html.js b/plugins/links/oembed-photo-html.js
index 5733b68d7..5f0907d49 100644
--- a/plugins/links/oembed-photo-html.js
+++ b/plugins/links/oembed-photo-html.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/links/oembed-video.js b/plugins/links/oembed-video.js
index c983761b0..bf96b981c 100644
--- a/plugins/links/oembed-video.js
+++ b/plugins/links/oembed-video.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 import * as entities from 'entities';
 
 export default {
diff --git a/plugins/meta/ld-article.js b/plugins/meta/ld-article.js
index 42ac59657..1a994145e 100644
--- a/plugins/meta/ld-article.js
+++ b/plugins/meta/ld-article.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/meta/ld-product.js b/plugins/meta/ld-product.js
index 595ffab93..bfb07cc77 100644
--- a/plugins/meta/ld-product.js
+++ b/plugins/meta/ld-product.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 
 export default {
 
diff --git a/plugins/meta/oembed-description.js b/plugins/meta/oembed-description.js
index 8608c8b5f..021f0d040 100644
--- a/plugins/meta/oembed-description.js
+++ b/plugins/meta/oembed-description.js
@@ -1,4 +1,5 @@
-import cheerio from 'cheerio';
+import cheerio_pkg from 'cheerio';
+const cheerio = cheerio_pkg.default;
 
 export default {
 

From 2fee8456a564f9905b39016d057455e87f4d42c5 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 22:34:32 +0300
Subject: [PATCH 035/102] fix imports

---
 lib/core.js               | 2 +-
 plugins/links/og-video.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index 6190b878e..90df33ce4 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -7,7 +7,7 @@
     import log from '../logging.js';
     import * as oembedUtils from './oembed.js';
     import * as pluginLoader from './loader/pluginLoader.js';
-    import * as requestWrapper from './request.js';
+    import requestWrapper from './request.js';
 
     const plugins = pluginLoader._plugins,
         providedParamsDict = pluginLoader._providedParamsDict,
diff --git a/plugins/links/og-video.js b/plugins/links/og-video.js
index 5e23eabe2..e690773f7 100644
--- a/plugins/links/og-video.js
+++ b/plugins/links/og-video.js
@@ -1,5 +1,5 @@
 import * as _ from "underscore";
-import * as utils from './utils.js';
+import utils from './utils.js';
 
 function getVideoLinks(video, whitelistRecord) {
     

From c5caf9bf8b6c6f85e69c4405c92098ab8d3f9b84 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 19 Oct 2021 22:38:48 +0300
Subject: [PATCH 036/102] fix audit

---
 yarn.lock | 98 +++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 32 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 15d64c6f7..db4311449 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -24,24 +24,31 @@
     fastq "^1.6.0"
 
 "@types/bson@*":
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.0.3.tgz#30889d2ffde6262abbe38659364c631454999fbf"
-  integrity sha512-mVRvYnTOZJz3ccpxhr3wgxVmSeiYinW+zlzQz3SXWaJmD1DuL05Jeq7nKw3SnbKmbleW5qrLG5vdyWe/A9sXhw==
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.2.0.tgz#a2f71e933ff54b2c3bf267b67fa221e295a33337"
+  integrity sha512-ELCPqAdroMdcuxqwMgUpifQyRoTpyYCNr1V9xKyF40VsBobsj+BbWNRvwGchMgBPGqkw655ypkjj2MEF5ywVwg==
+  dependencies:
+    bson "*"
+
+"@types/bson@1.x || 4.0.x":
+  version "4.0.5"
+  resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.0.5.tgz#9e0e1d1a6f8866483f96868a9b33bc804926b1fc"
+  integrity sha512-vVLwMUqhYJSQ/WKcE60eFqcyuWse5fGH+NMAXHuKrUAPoryq3ATxk5o4bgYNtg5aOM4APVg7Hnb3ASqUYG0PKg==
   dependencies:
     "@types/node" "*"
 
 "@types/mongodb@^3.5.27":
-  version "3.6.12"
-  resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.12.tgz#727960d34f35054d2f2ce68909e16094f742d935"
-  integrity sha512-49aEzQD5VdHPxyd5dRyQdqEveAg9LanwrH8RQipnMuulwzKmODXIZRp0umtxi1eBUfEusRkoy8AVOMr+kVuFog==
+  version "3.6.20"
+  resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.6.20.tgz#b7c5c580644f6364002b649af1c06c3c0454e1d2"
+  integrity sha512-WcdpPJCakFzcWWD9juKoZbRtQxKIMYF/JIAM4JrNHrMcnJL6/a2NWjXxW7fo9hxboxxkg+icff8d7+WIEvKgYQ==
   dependencies:
     "@types/bson" "*"
     "@types/node" "*"
 
 "@types/node@*":
-  version "15.0.2"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-15.0.2.tgz#51e9c0920d1b45936ea04341aa3e2e58d339fb67"
-  integrity sha512-p68+a+KoxpoB47015IeYZYRrdqMUcpbK8re/zpFB8Ld46LHC1lPEbp3EXgkEhAYEcPvjJF6ZO+869SQ0aH1dcA==
+  version "16.11.1"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.1.tgz#2e50a649a50fc403433a14f829eface1a3443e97"
+  integrity sha512-PYGcJHL9mwl1Ek3PLiYgyEKtwTMmkMw4vbiyz/ps3pfdRYLVv+SN7qHVAImrjdAXxgluDEw6Ph4lyv+m9UpRmA==
 
 "@types/tough-cookie@^4.0.0":
   version "4.0.1"
@@ -160,6 +167,11 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
+base64-js@^1.3.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
+  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
+
 bcrypt-pbkdf@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
@@ -226,6 +238,13 @@ browser-stdout@1.3.1:
   resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60"
   integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==
 
+bson@*:
+  version "4.5.3"
+  resolved "https://registry.yarnpkg.com/bson/-/bson-4.5.3.tgz#de3783b357a407d935510beb1fbb285fef43bb06"
+  integrity sha512-qVX7LX79Mtj7B3NPLzCfBiCP6RAsjiV8N63DjlaVVpZW+PFoDTxQ4SeDbSpcqgE6mXksM5CAwZnXxxxn/XwC0g==
+  dependencies:
+    buffer "^5.6.0"
+
 bson@^1.1.4:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.6.tgz#fb819be9a60cd677e0853aee4ca712a785d6618a"
@@ -236,6 +255,14 @@ buffer-shims@^1.0.0:
   resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51"
   integrity sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=
 
+buffer@^5.6.0:
+  version "5.7.1"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
+  integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
+  dependencies:
+    base64-js "^1.3.1"
+    ieee754 "^1.1.13"
+
 bytes@3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
@@ -460,11 +487,16 @@ delayed-stream@~1.0.0:
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
   integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk=
 
-denque@^1.1.0, denque@^1.4.1, denque@^1.5.0:
+denque@^1.1.0, denque@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de"
   integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==
 
+denque@^1.4.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf"
+  integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
+
 depd@~1.1.1, depd@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9"
@@ -931,10 +963,6 @@ http-errors@~1.8.0:
     statuses ">= 1.5.0 < 2"
     toidentifier "1.0.0"
 
-http-parser-js@itteco/http-parser-js#magicode-fix-22:
-  version "0.4.10"
-  resolved "https://codeload.github.com/itteco/http-parser-js/tar.gz/18b6b19b41fa3dd5d032fc2ffc58ac909da0a86e"
-
 http-signature@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
@@ -956,6 +984,11 @@ iconv-lite@0.4.24:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
+ieee754@^1.1.13:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
+  integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
+
 ignore@^5.1.8:
   version "5.1.8"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
@@ -1239,15 +1272,15 @@ moment@2.19.3:
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.3.tgz#bdb99d270d6d7fda78cc0fbace855e27fe7da69f"
   integrity sha1-vbmdJw1tf9p4zA+6zoVeJ/59pp8=
 
-mongodb@3.6.11:
-  version "3.6.11"
-  resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.6.11.tgz#8a59a0491a92b00a8c925f72ed9d9a5b054aebb2"
-  integrity sha512-4Y4lTFHDHZZdgMaHmojtNAlqkvddX2QQBEN0K//GzxhGwlI9tZ9R0vhbjr1Decw+TF7qK0ZLjQT292XgHRRQgw==
+mongodb@3.7.2:
+  version "3.7.2"
+  resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.7.2.tgz#d0d43b08ff1e5c13f4112175e321fa292cf35a3d"
+  integrity sha512-/Qi0LmOjzIoV66Y2JQkqmIIfFOy7ZKsXnQNlUXPFXChOw3FCdNqVD5zvci9ybm6pkMe/Nw+Rz9I0Zsk2a+05iQ==
   dependencies:
     bl "^2.2.1"
     bson "^1.1.4"
     denque "^1.4.1"
-    optional-require "^1.0.3"
+    optional-require "^1.1.8"
     safe-buffer "^5.1.2"
   optionalDependencies:
     saslprep "^1.0.0"
@@ -1258,16 +1291,17 @@ mongoose-legacy-pluralize@1.0.2:
   integrity sha512-Yo/7qQU4/EyIS8YDFSeenIvXxZN+ld7YdV9LqFVQJzTLye8unujAWPZ4NWKfFA+RNjh+wvTWKY9Z3E5XM6ZZiQ==
 
 mongoose@^5.13.7:
-  version "5.13.7"
-  resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-5.13.7.tgz#b6938390e93807a6f2ea91a054af2faa67f17218"
-  integrity sha512-ADIvftZ+KfoTALMZ0n8HvBlezFhcUd73hQaHQDwQ+3X+JZlqE47fUy9yhFZ2SjT+qzmuaCcIXCfhewIc38t2fQ==
+  version "5.13.12"
+  resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-5.13.12.tgz#45ad4d8f4b1782cc547e1fa1946608b016d7829f"
+  integrity sha512-ZEuZ3X/yop9XyOyuCYMz+oxJxXBclm9LIsjKHB0QX2eaNqKNqkvZFzkElbJCj8FDvYmBZFh0OFHlkREhtie6uA==
   dependencies:
+    "@types/bson" "1.x || 4.0.x"
     "@types/mongodb" "^3.5.27"
     bson "^1.1.4"
     kareem "2.3.2"
-    mongodb "3.6.11"
+    mongodb "3.7.2"
     mongoose-legacy-pluralize "1.0.2"
-    mpath "0.8.3"
+    mpath "0.8.4"
     mquery "3.2.5"
     ms "2.1.2"
     optional-require "1.0.x"
@@ -1276,10 +1310,10 @@ mongoose@^5.13.7:
     sift "13.5.2"
     sliced "1.0.1"
 
-mpath@0.8.3:
-  version "0.8.3"
-  resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.8.3.tgz#828ac0d187f7f42674839d74921970979abbdd8f"
-  integrity sha512-eb9rRvhDltXVNL6Fxd2zM9D4vKBxjVVQNLNijlj7uoXUy19zNDsIif5zR+pWmPCWNKwAtqyo4JveQm4nfD5+eA==
+mpath@0.8.4:
+  version "0.8.4"
+  resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.8.4.tgz#6b566d9581621d9e931dd3b142ed3618e7599313"
+  integrity sha512-DTxNZomBcTWlrMW76jy1wvV37X/cNNxPW1y2Jzd4DZkAaC5ZGsm8bfGfNOthcDuRJujXLqiuS6o3Tpy0JEoh7g==
 
 mquery@3.2.5:
   version "3.2.5"
@@ -1382,10 +1416,10 @@ optional-require@1.0.x:
   resolved "https://registry.yarnpkg.com/optional-require/-/optional-require-1.0.3.tgz#275b8e9df1dc6a17ad155369c2422a440f89cb07"
   integrity sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA==
 
-optional-require@^1.0.3:
-  version "1.1.7"
-  resolved "https://registry.yarnpkg.com/optional-require/-/optional-require-1.1.7.tgz#9ab5b254f59534108d4b2201d9ae96a063abc015"
-  integrity sha512-cIeRZocXsZnZYn+SevbtSqNlLbeoS4mLzuNn4fvXRMDRNhTGg0sxuKXl0FnZCtnew85LorNxIbZp5OeliILhMw==
+optional-require@^1.1.8:
+  version "1.1.8"
+  resolved "https://registry.yarnpkg.com/optional-require/-/optional-require-1.1.8.tgz#16364d76261b75d964c482b2406cb824d8ec44b7"
+  integrity sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA==
   dependencies:
     require-at "^1.0.6"
 

From b1e4e4e252ff704ce9b9f1ea98496d641cef7835 Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Wed, 20 Oct 2021 00:15:50 -0400
Subject: [PATCH 037/102] reivew the use of cheerio and query-strings

---
 lib/plugins/system/oembed/oembed.js           |  17 +-
 plugins/custom/query.js                       |  17 ++
 plugins/domains/art19.com.js                  |  89 ++++------
 .../brightcove.com/players.brightcove.net.js  |  29 ++--
 plugins/domains/cartodb.com.js                |  31 +---
 plugins/domains/codepen.io.js                 |  87 ++++------
 plugins/domains/mixcloud.com.js               | 163 ++++++++----------
 .../domains/pinterest.com/pinterest.pin.js    |   3 +-
 plugins/domains/scribd.com/scribd.com.js      |  63 ++-----
 plugins/domains/slideshare.net.js             |  81 ++++-----
 .../domains/soundcloud.com/soundcloud.com.js  |  18 +-
 plugins/domains/speakerdeck.com.js            |  54 ++----
 plugins/domains/spotify.com.js                |  11 +-
 plugins/domains/strawpoll.me.js               |   6 +-
 plugins/domains/tumblr.com/tumblr.api.js      |   5 +-
 plugins/links/hosted/23video-hosted.js        |  48 +-----
 plugins/links/oembed-photo-html.js            |  15 +-
 plugins/links/oembed-video.js                 |  27 +--
 plugins/meta/ld-article.js                    |   2 +-
 19 files changed, 287 insertions(+), 479 deletions(-)
 create mode 100644 plugins/custom/query.js

diff --git a/lib/plugins/system/oembed/oembed.js b/lib/plugins/system/oembed/oembed.js
index d7d7ea4ef..a6b2213a8 100644
--- a/lib/plugins/system/oembed/oembed.js
+++ b/lib/plugins/system/oembed/oembed.js
@@ -51,11 +51,20 @@ function _getOembedIframe(oembed) {
             oembed._iframe = fixOembedIframeAttributes($iframe[0].attribs);
 
             if (oembed._iframe && oembed._iframe.src) {
-                oembed._iframe.query = URL.parse(oembed._iframe.src, true).query;
+                var src = URL.parse(oembed._iframe.src, true);
+                oembed._iframe.host = src.host;
+                oembed._iframe.pathname = src.pathname;
+                oembed._iframe.path = src.path;
+                oembed._iframe.query = src.query;
+                oembed._iframe.placeholder = oembed.thumbnail_url;
                 oembed._iframe.replaceQuerystring = function(params) {
                     var qs = querystring.stringify({...oembed._iframe.query, ...params});
                     return oembed._iframe.src.replace(/\?.*$/, '') + (qs ? '?' + qs : '');
                 }
+                oembed._iframe.assignQuerystring = function(params) {
+                    var qs = querystring.stringify(params);
+                    return oembed._iframe.src.replace(/\?.*$/, '') + (qs ? '?' + qs : '');
+                }                
             }
         } else {
             oembed._iframe = null;
@@ -82,7 +91,7 @@ function getOembedIframeAttr(oembed) {
 
 export default {
 
-    provides: ['self', 'oembedError'],
+    provides: ['self', 'oembedError', 'iframe'],
 
     getData: function(url, oembedLinks, options, cb) {
 
@@ -130,6 +139,10 @@ export default {
                 }, oembed),
             };
 
+            if (oembed.html && _getOembedIframe(oembed)) {
+                result.iframe = _getOembedIframe(oembed)
+            } 
+
             // If no oEmbed record for the domain - allow to be whitelisted by the oEmbed endpoint domian record.
             if (options.getWhitelistRecord) {
                 var currentWhitelistRecord = options.getWhitelistRecord(url, {disableWildcard: true});
diff --git a/plugins/custom/query.js b/plugins/custom/query.js
new file mode 100644
index 000000000..ace40f385
--- /dev/null
+++ b/plugins/custom/query.js
@@ -0,0 +1,17 @@
+import * as URL from "url"
+
+export default {
+
+    provides: 'query',
+
+    getData: function(url) {
+
+        if (/\?/i.test(url)) {
+            return {
+                query: URL.parse(url, true).query
+            };
+        } else {
+            return {query: {}}
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/domains/art19.com.js b/plugins/domains/art19.com.js
index 69ee1274d..7ca36544c 100644
--- a/plugins/domains/art19.com.js
+++ b/plugins/domains/art19.com.js
@@ -1,8 +1,3 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
-import * as querystring from 'querystring';
-import * as URL from "url";
-
 export default {
 
     re: [
@@ -18,65 +13,51 @@ export default {
         "domain-icon"
     ],
 
-    getLink: function(oembed, options) {
-
-        if (oembed.html) {
-
-            var $container = $('<div>');
-            try {
-                $container.html(oembed.html);
-            } catch(ex) {}
-
-            var $iframe = $container.find('iframe');
-
-            if ($iframe.length == 1) {
+    getLink: function(iframe, options) {
 
-                var player = $iframe.attr('src');
-                var params = URL.parse(player, true).query;
+        var params = Object.assign(iframe.query);
 
-                var theme = options.getRequestOptions('players.theme', 'light');
-                params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue';
+        var theme = options.getRequestOptions('players.theme', 'light');
+        params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue';
 
-                var opts = {};
+        var opts = {};
 
-                var horizontal = options.getRequestOptions('players.horizontal', true);
+        var horizontal = options.getRequestOptions('players.horizontal', true);
 
-                if (horizontal) {
-                    delete params.type;
-                    delete params.stretch;
+        if (horizontal) {
+            delete params.type;
+            delete params.stretch;
 
-                    var theme = options.getRequestOptions('players.theme', 'light');                    
-                    params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue';
+            var theme = options.getRequestOptions('players.theme', 'light');                    
+            params.theme = theme === 'light' ? 'light-gray-blue' : 'dark-blue';
 
-                    opts.theme = {
-                        label: CONFIG.L.theme,
-                        value: theme,
-                        values: {
-                            light: CONFIG.L.light,
-                            dark: CONFIG.L.dark
-                        }
-                    };
-                } else {
-                    params.type = 'artwork';
-                    params.stretch = true;
-                    delete params.theme;
-                }
-
-                opts.horizontal = {
-                    label: CONFIG.L.horizontal,
-                    value: horizontal
+            opts.theme = {
+                label: CONFIG.L.theme,
+                value: theme,
+                values: {
+                    light: CONFIG.L.light,
+                    dark: CONFIG.L.dark
                 }
+            };
+        } else {
+            params.type = 'artwork';
+            params.stretch = true;
+            delete params.theme;
+        }
 
-                return {
-                    href: (/\?/.test(player) ? player.replace(/\?.+/, '?') : player + '?') + querystring.stringify(params),
-                    type: CONFIG.T.text_html,
-                    rel: [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.oembed], // keep rel oembed here - it prevents validators from removing embed srcz
-                    media: horizontal ? {height: oembed.height, scrolling: 'no'} : {'aspect-ratio': 1},
-                    scrolling: 'no',
-                    options: opts
-                };
-            }
+        opts.horizontal = {
+            label: CONFIG.L.horizontal,
+            value: horizontal
         }
+
+        return {
+            href: iframe.assignQuerystring(params),
+            type: CONFIG.T.text_html,
+            rel: [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.oembed], // keep rel oembed here - it prevents validators from removing embed srcz
+            media: horizontal ? {height: iframe.height, scrolling: 'no'} : {'aspect-ratio': 1},
+            scrolling: 'no',
+            options: opts
+        };
     },
 
     tests: [{
diff --git a/plugins/domains/brightcove.com/players.brightcove.net.js b/plugins/domains/brightcove.com/players.brightcove.net.js
index 6136dbf8c..90a74d64b 100644
--- a/plugins/domains/brightcove.com/players.brightcove.net.js
+++ b/plugins/domains/brightcove.com/players.brightcove.net.js
@@ -1,5 +1,3 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
 import * as utils from '../../../lib/utils.js';
 
 export default {
@@ -17,11 +15,11 @@ export default {
     ],
 
     //HTML parser will 404 if BC account or player does not exist.
-    getLinks: function(url, oembed, options, cb) {
+    getLinks: function(url, iframe, options, cb) {
 
         var player = {
             type: CONFIG.T.text_html,
-            rel: [CONFIG.R.oembed, CONFIG.R.player, CONFIG.R.html5]
+            rel: [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.oembed]
         };
 
         // autoplay=true comes from `brightcove-in-page-promo` only and follows whitelistRecord
@@ -31,15 +29,8 @@ export default {
             player.autoplay = "autoplay=true";
         }
 
-        var $container = cheerio('<div>');
-        try {
-            $container.html(oembed.html);
-        } catch (ex) {}
-
-        var $iframe = $container.find('iframe');
-
-        if ($iframe.length == 1) {
-            player.href = $iframe.attr('src') + (/&autoplay=true/.test(url) ? '&autoplay=true' : ''); // autoplay=true in URL comes from brightcove-allow-in-page whitelist record            
+        if (iframe.src) {
+            player.href = iframe.src + (/&autoplay=true/.test(url) ? '&autoplay=true' : ''); // autoplay=true in URL comes from brightcove-allow-in-page whitelist record            
         }
 
         if (/&iframe-url=/.test(url)) {
@@ -50,9 +41,9 @@ export default {
             player.accept = CONFIG.T.text_html; // verify that it exists and isn't X-Frame-Optioned            
         }
 
-        if (oembed.thumbnail_url) {
+        if (iframe.placeholder) {
 
-            utils.getImageMetadata(oembed.thumbnail_url, options, function(error, data) {
+            utils.getImageMetadata(iframe.placeholder, options, function(error, data) {
 
                 var links = [];
 
@@ -63,7 +54,7 @@ export default {
                 } else if (data.width && data.height) {
 
                     links.push({
-                        href: oembed.thumbnail_url,
+                        href: iframe.placeholder,
                         type: CONFIG.T.image, 
                         rel: CONFIG.R.thumbnail,
                         width: data.width,
@@ -71,14 +62,14 @@ export default {
                     });                    
                 }
 
-                player['aspect-ratio'] = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height;
+                player['aspect-ratio'] = (data.width && data.height) ? data.width / data.height : iframe.width / iframe.height;
                 links.push(player);
 
-                cb(null, links);
+                return cb(null, links);
 
             });
         } else {
-            cb (null, player);
+            return cb (null, player);
         }
 
     },
diff --git a/plugins/domains/cartodb.com.js b/plugins/domains/cartodb.com.js
index 2d172e72e..129f7236b 100644
--- a/plugins/domains/cartodb.com.js
+++ b/plugins/domains/cartodb.com.js
@@ -1,6 +1,3 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
-
 export default {
 
     re: /^https?:(\/\/[\w-]+\.carto(?:db)?\.com\/(?:u\/[\w-]+\/)?viz\/[a-z0-9-]+)/i,
@@ -24,27 +21,15 @@ export default {
         };
     },
 
-    getLink: function(oembed) {
-
-        var $container = cheerio('<div>');
-        try {
-            $container.html(oembed.html5 || oembed.html);
-        } catch (ex) {}
-
-        var $iframe = $container.find('iframe');
-
-        if ($iframe.length == 1) {
-
-            return {
-                href: $iframe.attr('src'),
-                type: CONFIG.T.text_html,
-                rel: [CONFIG.R.app, CONFIG.R.ssl, CONFIG.R.html5],
-                "aspect-ratio": 4/3,
-                "padding-bottom": 30
-                // aspect 4:3 is better than height=520px and width=100%
-            };
+    getLink: function(iframe) {
+        return {
+            href: iframe.src,
+            type: CONFIG.T.text_html,
+            rel: [CONFIG.R.app, CONFIG.R.ssl, CONFIG.R.html5],
+            "aspect-ratio": 4/3,
+            "padding-bottom": 30
+            // aspect 4:3 is better than height=520px and width=100%
         }
-
     },
 
     tests: [{
diff --git a/plugins/domains/codepen.io.js b/plugins/domains/codepen.io.js
index d13b2f7d3..006b02b62 100644
--- a/plugins/domains/codepen.io.js
+++ b/plugins/domains/codepen.io.js
@@ -1,8 +1,3 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
-import * as querystring from 'querystring';
-import * as URL from "url";
-
 export default {
 
     re: /https?:\/\/codepen\.io\/(?:[a-z0-9\-_]+\/)?(pen|details|full)\/([a-z0-9\-]+)/i,
@@ -16,7 +11,7 @@ export default {
         "domain-icon"
     ],
 
-    getLink: function(oembed, options) {
+    getLink: function(oembed, iframe, options) {
 
         if (oembed.author_url ===  "https://codepen.io/anon/") {
             return { // And no fallback to generics
@@ -24,57 +19,47 @@ export default {
             }
         }
 
-        var $container = $('<div>');
-        try{
-            $container.html(oembed.html);
-        } catch(ex) {}
-
-        var $iframe = $container.find('iframe');
-
-        if ($iframe.length == 1) {
-
-            var href = $iframe.attr('src');
-            var params = URL.parse(href, true).query;
+        var params = Object.assign(iframe.query);
 
-            var click_to_load = options.getRequestOptions('codepen.click_to_load', /\/embed\/preview\//.test(href));
-            href = href.replace(/\/embed\/(?:preview\/)?/, '/embed/').replace(/\/embed\//, '/embed/' + (click_to_load ? 'preview/' : ''));
+        params.height = options.getRequestOptions('codepen.height', oembed.height);
 
-            params.height = options.getRequestOptions('codepen.height', oembed.height);
+        var theme = options.getRequestOptions('players.theme', params.theme || 'auto');
 
-            var theme = options.getRequestOptions('players.theme', params.theme || 'auto');
-
-            if (theme === 'auto') {
-                delete params['theme-id'];
-            } else {
-                params['theme-id'] = theme;
-            }
+        if (theme === 'auto') {
+            delete params['theme-id'];
+        } else {
+            params['theme-id'] = theme;
+        }
 
-            return {
-                href: href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1'),
-                type: CONFIG.T.text_html,
-                rel: [CONFIG.R.app, CONFIG.R.oembed, CONFIG.R.html5],
-                height: params.height,
-                options: {
-                    height: {
-                        label: CONFIG.L.height,
-                        value: params.height,
-                        placeholder: 'ex.: 600, in px'
-                    },
-                    click_to_load: {
-                        label: 'Use click-to-load',
-                        value: click_to_load
-                    },
-                    theme: {
-                        label: CONFIG.L.theme,
-                        value: theme,
-                        values: {
-                            light: CONFIG.L.light,
-                            dark: CONFIG.L.dark,
-                            auto: CONFIG.L.default
-                        }
+        var href = iframe.assignQuerystring(params);
+        var click_to_load = options.getRequestOptions('codepen.click_to_load', /\/embed\/preview\//.test(href));
+        href = href.replace(/\/embed\/(?:preview\/)?/, '/embed/').replace(/\/embed\//, '/embed/' + (click_to_load ? 'preview/' : ''));
+
+        return {
+            href: href,
+            type: CONFIG.T.text_html,
+            rel: [CONFIG.R.app, CONFIG.R.oembed, CONFIG.R.html5],
+            height: params.height,
+            options: {
+                height: {
+                    label: CONFIG.L.height,
+                    value: params.height,
+                    placeholder: 'ex.: 600, in px'
+                },
+                click_to_load: {
+                    label: 'Use click-to-load',
+                    value: click_to_load
+                },
+                theme: {
+                    label: CONFIG.L.theme,
+                    value: theme,
+                    values: {
+                        light: CONFIG.L.light,
+                        dark: CONFIG.L.dark,
+                        auto: CONFIG.L.default
                     }
                 }
-            };
+            }
         }
     },
 
diff --git a/plugins/domains/mixcloud.com.js b/plugins/domains/mixcloud.com.js
index 3774907a2..9e3a918e6 100644
--- a/plugins/domains/mixcloud.com.js
+++ b/plugins/domains/mixcloud.com.js
@@ -1,8 +1,3 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
-import * as querystring from 'querystring';
-import * as URL from "url";
-
 export default {
 
     re: [
@@ -19,10 +14,10 @@ export default {
         "domain-icon"
     ],
 
-    getLink: function (oembed, whitelistRecord, options) {
+    getLink: function (oembed, iframe, whitelistRecord, options) {
 
         // let Whitelist/default fallbacks take control if oEmbed fails
-        if (!(oembed.type === "rich" && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.rich'))) {
+        if (!(oembed.type === "rich" && iframe.src && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.rich'))) {
             return;
         } else {
 
@@ -31,92 +26,82 @@ export default {
                 type: CONFIG.T.text_html
             };
 
-            var $container = cheerio('<div>');
-            try {
-                $container.html(oembed.html);
-            } catch (ex) {}
-
-            var $iframe = $container.find('iframe');
-
-            if ($iframe.length == 1) {
-
-                var href = $iframe.attr('src');
-
-                if (/\/widget\/follow\//.test(href)) {
-                    widget.href = href;
-                    widget.rel.push(CONFIG.R.summary);
-                    widget.width = oembed.width;
-                    widget.height = oembed.height;
-
-                    // widget.error = 'Mixcloud user summary is currently broken'; // Sept 22, 2020 - works as of Jan 25, 2021
-
-                } else {
-
-                    var params = URL.parse(href, true).query;
-                    if (options.getProviderOptions('players.horizontal') === false) {
-                        delete params.hide_cover;
-                    }
-                    var style = options.getRequestOptions('mixcloud.style', params.mini == 1 ? 'mini' : (params.hide_cover == 1 ? 'classic' : 'cover'));
-                    var theme = options.getRequestOptions('players.theme', params.light == 1 ? 'light' : 'dark');
-
-                    if (theme === 'light') {
-                        params.light = 1;
-                    }
-
-                    if (options.getRequestOptions('mixcloud.hide_artwork', params.hide_artwork)) {
-                        params.hide_artwork = 1;
-                    }                    
-
-                    if (style === 'mini') {
-                        params.mini = 1;
-                        params.hide_cover = 1;
-                    } else if (style === 'classic') {
-                        delete params.mini;
-                        params.hide_cover = 1;
-                    } else if (style === 'cover') {
-                        delete params.mini;
-                        delete params.hide_cover;
-                        delete params.light;
-                        delete params.hide_artwork;
+            var href = iframe.src;
+
+            if (/\/widget\/follow\//.test(href)) {
+                widget.href = href;
+                widget.rel.push(CONFIG.R.summary);
+                widget.width = oembed.width;
+                widget.height = oembed.height;
+
+                // widget.error = 'Mixcloud user summary is currently broken'; // Sept 22, 2020 - works as of Jan 25, 2021
+
+            } else {
+
+                var params = Object.assign(iframe.query);
+                if (options.getProviderOptions('players.horizontal') === false) {
+                    delete params.hide_cover;
+                }
+                var style = options.getRequestOptions('mixcloud.style', params.mini == 1 ? 'mini' : (params.hide_cover == 1 ? 'classic' : 'cover'));
+                var theme = options.getRequestOptions('players.theme', params.light == 1 ? 'light' : 'dark');
+
+                if (theme === 'light') {
+                    params.light = 1;
+                }
+
+                if (options.getRequestOptions('mixcloud.hide_artwork', params.hide_artwork)) {
+                    params.hide_artwork = 1;
+                }                    
+
+                if (style === 'mini') {
+                    params.mini = 1;
+                    params.hide_cover = 1;
+                } else if (style === 'classic') {
+                    delete params.mini;
+                    params.hide_cover = 1;
+                } else if (style === 'cover') {
+                    delete params.mini;
+                    delete params.hide_cover;
+                    delete params.light;
+                    delete params.hide_artwork;
+                }
+
+                widget.href = iframe.assignQuerystring(params);
+                widget.autoplay = 'autoplay=1';                    
+
+                // mixcloud ignores &mini=1 if there's no &hide_cover=1.
+                widget.height = !/&?hide_cover=1/i.test(widget.href) ? 400 : (/&?mini=1/i.test(widget.href) ? 60 : 120);
+                widget.scrolling = "no";
+                widget.rel.push(CONFIG.R.player);
+                widget.rel.push(CONFIG.R.auido);
+
+                widget.options = {
+                    style: {
+                        label: 'Widget style',
+                        value: style,
+                        values: {
+                            'mini': 'Mini',
+                            'classic': 'Classic',
+                            'cover': 'Picture'
+                        }
                     }
-
-                    widget.href = href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1');
-                    widget.autoplay = 'autoplay=1';                    
-
-                    // mixcloud ignores &mini=1 if there's no &hide_cover=1.
-                    widget.height = !/&?hide_cover=1/i.test(widget.href) ? 400 : (/&?mini=1/i.test(widget.href) ? 60 : 120);
-                    widget.scrolling = "no";
-                    widget.rel.push(CONFIG.R.player);
-                    widget.rel.push(CONFIG.R.auido);
-
-                    widget.options = {
-                        style: {
-                            label: 'Widget style',
-                            value: style,
-                            values: {
-                                'mini': 'Mini',
-                                'classic': 'Classic',
-                                'cover': 'Picture'
-                            }
+                };
+
+                if (style !== 'cover') {
+                    widget.options.theme = {
+                        label: CONFIG.L.theme,
+                        value: theme,
+                        values: {
+                            light: CONFIG.L.light,
+                            dark: CONFIG.L.dark
                         }
                     };
+                    widget.options.hide_artwork = {
+                        label: CONFIG.L.hide_artwork,
+                        value: params.hide_artwork === 1
+                    };
 
-                    if (style !== 'cover') {
-                        widget.options.theme = {
-                            label: CONFIG.L.theme,
-                            value: theme,
-                            values: {
-                                light: CONFIG.L.light,
-                                dark: CONFIG.L.dark
-                            }
-                        };
-                        widget.options.hide_artwork = {
-                            label: CONFIG.L.hide_artwork,
-                            value: params.hide_artwork === 1
-                        };
-
-                    }
-                }                
+                }          
 
                 return [widget, {
                     href: oembed.image, 
diff --git a/plugins/domains/pinterest.com/pinterest.pin.js b/plugins/domains/pinterest.com/pinterest.pin.js
index 564c73cbb..e1f67efc0 100644
--- a/plugins/domains/pinterest.com/pinterest.pin.js
+++ b/plugins/domains/pinterest.com/pinterest.pin.js
@@ -4,8 +4,7 @@ export default {
     re: /^(https?:\/\/(?:\w{2,3}\.)?pinterest(?:\.com?)?\.\w{2,3})\/pin\/(?:[^\/]+\-)?(\d+)/i,
 
     mixins: [
-        "*",
-        "oembed-iframe"
+        "*"
     ],
 
     // https://developers.pinterest.com/tools/widget-builder/?type=pin&terse=true&size=large
diff --git a/plugins/domains/scribd.com/scribd.com.js b/plugins/domains/scribd.com/scribd.com.js
index b4d3504a6..f2d24749c 100644
--- a/plugins/domains/scribd.com/scribd.com.js
+++ b/plugins/domains/scribd.com/scribd.com.js
@@ -1,23 +1,14 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
-import * as utils from '../../../lib/utils.js';
-import * as querystring from 'querystring';
-import * as URL from "url";
-
 export default {
 
     re: [
         /^https?:\/\/(?:www|\w{2})\.scribd\.com\/(doc|document|embeds|presentation|fullscreen)\/(\d+)/i
     ],
 
-    provides: ['scribdData'],
-
-    mixins: [ "*" ],
+    mixins: [ "*", "query"],
 
-    getLink: function(url, scribdData, options) {
-            var href = scribdData.href;
-            var params = URL.parse(href, true).query;
-            var hash = URL.parse(url, true).hash;
+    getLink: function(url, iframe, query, options) {
+            var params = Object.assign(iframe.query);
+            var hash = query.hash;
 
             var slideshow = options.getRequestOptions('scribd.slideshow', params.view_mode === 'slideshow');
             if (slideshow) {
@@ -37,10 +28,10 @@ export default {
             }
 
             return {
-                href: href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1'),
+                href: iframe.assignQuerystring(params),
                 accept: CONFIG.T.text_html,
                 rel: slideshow ? [CONFIG.R.player, CONFIG.R.slideshow, CONFIG.R.html5, CONFIG.R.oembed] : [CONFIG.R.reader, CONFIG.R.html5, CONFIG.R.oembed],
-                'aspect-ratio': scribdData.aspect,
+                'aspect-ratio': iframe['data-aspect-ratio'],
                 'padding-bottom': 45, // toolbar
                 options: {
                     slideshow: {
@@ -56,44 +47,10 @@ export default {
         }
     },
 
-    getData: function(urlMatch, og, oembed, options, cb) {
-
-        if (!og.image) {
-            return 'embeds' === urlMatch[1]
-                ? cb({redirect: `https://www.scribd.com/document/${urlMatch[2]}`})
-                : cb(null, null);
-        }
-
-        utils.getImageMetadata(og.image.value || og.image, options, function(error, data) {
-
-            if (error || data.error) {
-                console.log ('Error getting preview for Scribd: ' + error);
-            } else {
-                var $container = $('<div>');
-                try {
-                    $container.html(oembed.html);
-                } catch(ex) {}
-
-                var $iframe = $container.find('iframe');
-                if ($iframe.length === 1) {
-
-                    return cb(null, {
-                        scribdData: {
-                            aspect:
-                                data.width
-                                && data.height
-                                ? data.width / data.height
-                                : (oembed.thumbnail_height ? oembed.thumbnail_width / oembed.thumbnail_height : null),
-
-                            href: $iframe.attr('src')
-                        }
-                    })
-
-                } else {
-                    return cb(null, null)
-                }
-            }
-        });
+    getData: function(urlMatch, options, cb) {
+        return 'embeds' === urlMatch[1]
+            ? cb({redirect: `https://www.scribd.com/document/${urlMatch[2]}`})
+            : cb(null, null);
     },
 
     tests: [{
diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js
index 92bc96b00..026933b2b 100644
--- a/plugins/domains/slideshare.net.js
+++ b/plugins/domains/slideshare.net.js
@@ -1,6 +1,4 @@
 import * as utils from '../../lib/utils.js';
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
 
 export default {
 
@@ -31,62 +29,47 @@ export default {
 
     },
 
-    getLink: function(oembed, options, cb) {
+    getLink: function(oembed, iframe, options, cb) {
 
-        if (oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) {
-            var links = [];
+        if (iframe.src && oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) {
 
             var firstSlide = (/^\/\//.test(oembed.slide_image_baseurl) ? 'http:' : '') + oembed.slide_image_baseurl + '1' + oembed.slide_image_baseurl_suffix;
 
             utils.getImageMetadata(firstSlide, options, function(error, data) {
 
-                if (error || data.error) {
-
-                    console.log ('Error getting first slide for Slideshare: ' + error);
-
-                } else if (data.width && data.height) {
-
-                    links.push({
-                        href: firstSlide,
-                        type: CONFIG.T.image, 
-                        rel: CONFIG.R.thumbnail,
-                        width: data.width,
-                        height: data.height
-                    });
-                }
-
-                var $container = $('<div>');
-                try {
-                    $container.html(oembed.html);
-                } catch(ex) {}
-
-                var $iframe = $container.find('iframe');
-
-                var aspect = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height;
-
-                if ($iframe.length == 1) {
-                    links.push({
-                        href: $iframe.attr('src').replace('http:', ''),
-                        type: CONFIG.T.text_html,
-                        rel: [aspect > 1 ? CONFIG.R.player : CONFIG.R.reader, CONFIG.R.slideshow, CONFIG.R.html5],
-                        "aspect-ratio": aspect,
-                        "padding-bottom": 38
-                    });
+                if (error || data.error || !data.width || !data.height) {
+
+                    return cb('Error getting first slide for Slideshare: ' + error);
+
+                } else {
+
+                    var aspect = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height;
+
+                    return cb(null, [{
+                            href: firstSlide,
+                            type: CONFIG.T.image, 
+                            rel: CONFIG.R.thumbnail,
+                            width: data.width,
+                            height: data.height
+                        }, {
+                            href: oembed.thumbnail,
+                            type: CONFIG.T.image,
+                            rel: [CONFIG.R.thumbnail, CONFIG.R.oembed],
+                            width: oembed.thumbnail_width,
+                            height: data.height ? Math.round (oembed.thumbnail_width / (data.width / data.height)) : oembed.thumbnail_height
+                        }, {
+                            href: iframe.src,
+                            type: CONFIG.T.text_html,
+                            rel: [aspect > 1 ? CONFIG.R.player : CONFIG.R.reader, CONFIG.R.slideshow, CONFIG.R.html5],
+                            "aspect-ratio": aspect,
+                            "padding-bottom": 38
+                        }
+                    ]);
                 }
 
-                links.push ({
-                    href: oembed.thumbnail,
-                    type: CONFIG.T.image,
-                    rel: [CONFIG.R.thumbnail, CONFIG.R.oembed],
-                    width: oembed.thumbnail_width,
-                    height: data.height ? Math.round (oembed.thumbnail_width / (data.width / data.height)) : oembed.thumbnail_height
-                });
-
-                cb(null, links);
-
             });
         } else {
-            cb (null, null);
+            cb(null, null);
         }
 
     },
@@ -105,7 +88,7 @@ export default {
         page: "http://www.slideshare.net/popular/today",
         selector: "a.iso_slideshow_link"
     }, {skipMethods: ["getData"]},
-        "http://www.slideshare.net/geniusworks/gamechangers-the-next-generation-of-business-innovation-by-peter-fisk#btnNext",
+        "https://www.slideshare.net/DataReportal/digital-2020-global-digital-overview-january-2020-v01-226017535",
         "https://www.slideshare.net/EnjoyDigitAll/le-design-thinking-by-enjoydigitall-71136562"
     ]
 };
\ No newline at end of file
diff --git a/plugins/domains/soundcloud.com/soundcloud.com.js b/plugins/domains/soundcloud.com/soundcloud.com.js
index 536c6b3f3..905a2365a 100644
--- a/plugins/domains/soundcloud.com/soundcloud.com.js
+++ b/plugins/domains/soundcloud.com/soundcloud.com.js
@@ -1,11 +1,6 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
-import * as querystring from 'querystring';
-import * as URL from "url";
-
 export default {
 
-    provides: ['__allow_soundcloud_meta', 'sound', 'iframe'],
+    provides: ['__allow_soundcloud_meta', 'sound'],
 
     mixins: [
         "oembed-title",
@@ -22,8 +17,7 @@ export default {
 
         if (iframe.src && sound.count !== 0) {
 
-            var href = iframe.src;
-            var params = URL.parse(href, true).query;
+            var params = Object.assign(iframe.query);
 
             if (options.getRequestOptions('players.horizontal', options.getProviderOptions('soundcloud.old_player') || options.getProviderOptions(CONFIG.O.less))) {
                 params.visual = false;
@@ -38,7 +32,7 @@ export default {
                 params.color = options.getProviderOptions('soundcloud.color');
             }
 
-            href = href.replace(/\?.+/, '') + querystring.stringify(params).replace(/^(.)/, '?$1');
+            var href = iframe.assignQuerystring(params);
             var height = options.getRequestOptions('soundcloud.height', options.getProviderOptions('players.horizontal') === false ? 0 : (/visual=false/.test(href) ? 166 : iframe.height));
             // Fallback to old values.
             if (height === 'auto') {
@@ -119,8 +113,7 @@ export default {
             ) || !oembed.description)
         ) {
             return {
-                __allow_soundcloud_meta: true,
-                iframe: oembed.getIframe()
+                __allow_soundcloud_meta: true
             }
             
         } else {
@@ -133,8 +126,7 @@ export default {
                         width: oembed.thumbnail_width,
                         height: oembed.thumbnail_height
                     }
-                },
-                iframe: oembed.getIframe()
+                }
             }
         }
     },
diff --git a/plugins/domains/speakerdeck.com.js b/plugins/domains/speakerdeck.com.js
index 8cd6a3c15..587ba1143 100644
--- a/plugins/domains/speakerdeck.com.js
+++ b/plugins/domains/speakerdeck.com.js
@@ -1,54 +1,34 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
-
 export default {
 
     mixins: [
-        "oembed-title",
-        "oembed-site",
-        "oembed-author",
-        "domain-icon",
-        "og-description"     
+        "*", "query"
     ],
 
-    getLink: function (url, oembed) {
-        var $container = $('<div>');
-        try {
-            $container.html(oembed.html);
-        } catch(ex) {}
+    getLink: function (url, iframe, query, options) {
+
+        var slide = options.getRequestOptions('speakerdeck.slide', query.slide ? parseInt(query.slide) || 1 : 1);
 
-        var $iframe = $container.find('iframe');
-        var doc; 
+        console.log('slide', slide);
 
-        if ($iframe.length == 1) {
-            var href = $iframe.attr('src').replace(/^\/\//, 'https://');
-            if (/\?slide=\d+/i.test(url)) {
+        if (iframe.src && iframe.width && iframe.height) {
+            var href = iframe.src.replace(/^\/\//, 'https://');
+            if (slide > 1) {
                 href +=  href.indexOf('?') > -1 ? '&' : '?';
-                href += url.match(/\?(slide=\d+)/i)[1];
+                href += 'slide=' + slide;
             }
-            doc = {
+            return {
                 href: href,
                 type: CONFIG.T.text_html,
                 rel: [CONFIG.R.player, CONFIG.R.html5],
-                "aspect-ratio": oembed.width / oembed.height 
-            }
-        }
-
-        var thumbnail;
-
-        if (doc) {
-            var id = doc.href.match(/\/\/speakerdeck\.com\/player\/([a-z0-9\-]+)/i)[1];
-
-            if (id) {
-                thumbnail = {
-                    href: 'https://speakerd.s3.amazonaws.com/presentations/' + id + '/slide_0.jpg',
-                    type: CONFIG.T.image,
-                    rel: [CONFIG.R.thumbnail]
-                };
+                "aspect-ratio": iframe.width / iframe.height,
+                options: {
+                    slide: {
+                        label: CONFIG.L.page,
+                        value: slide
+                    }
+                }
             }
         }
-
-        return [doc, thumbnail];
     },
 
 
diff --git a/plugins/domains/spotify.com.js b/plugins/domains/spotify.com.js
index fa5856b10..08414f13e 100644
--- a/plugins/domains/spotify.com.js
+++ b/plugins/domains/spotify.com.js
@@ -6,7 +6,6 @@ export default {
 
     mixins: [
         "oembed-title",
-        "oembed-iframe",
         "og-image",
         "oembed-thumbnail",
         "domain-icon"
@@ -28,18 +27,16 @@ export default {
 
         if (iframe.src) {
 
-            var src = iframe.src;
-
             var horizontal_player = options.getRequestOptions('players.horizontal', options.getProviderOptions(CONFIG.O.less));
 
             var player = {
-                href: src,
+                href: iframe.src,
                 type: CONFIG.T.text_html,
                 rel: [CONFIG.R.player, CONFIG.R.ssl, CONFIG.R.html5],
                 options: {}
             };
 
-            if (/album|playlist/.test(src)) {
+            if (/album|playlist/.test(iframe.src)) {
                 var include_playlist = options.getRequestOptions('spotify.playlist', true);
                 player.rel.push(CONFIG.R.playlist);
                 player.options.playlist = {
@@ -55,8 +52,8 @@ export default {
                     };
 
                 // Temp fix for broken v2 playlist.
-                player.href = src.replace(/\/embed\/playlist\-v2\//, '/embed/playlist/');
-            } else if (/episode|show/.test(src)) {
+                player.href = iframe.src.replace(/\/embed\/playlist\-v2\//, '/embed/playlist/');
+            } else if (/episode|show/.test(iframe.src)) {
                 player.rel.push(CONFIG.R.audio);
                 player.height = iframe.height || 232;
             } else {
diff --git a/plugins/domains/strawpoll.me.js b/plugins/domains/strawpoll.me.js
index f3fff1a36..d270a4ca5 100644
--- a/plugins/domains/strawpoll.me.js
+++ b/plugins/domains/strawpoll.me.js
@@ -1,5 +1,3 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
 import * as entities from 'entities';
 
 export default {
@@ -11,7 +9,7 @@ export default {
     getLink: function(urlMatch, cheerio) {
 
         var embed = entities.decodeHTML(cheerio('#embed-field-' + urlMatch[1]).attr('value'));
-        var iframe = $('<div>').html(embed).children('iframe');
+        var iframe = cheerio('<div>').html(embed).children('iframe');
         var height = parseInt(iframe.css('height').replace(/px$/, ''), 10) || 300;
         var width = parseInt(iframe.css('width').replace(/px$/, ''), 10) || 690;
 
@@ -27,7 +25,7 @@ export default {
     },
 
     tests: [
-        "http://www.strawpoll.me/1696",
+        "https://www.strawpoll.me/1696",
         "https://strawpoll.me/136"
     ]
 };
diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js
index 079399b75..fce4b181a 100644
--- a/plugins/domains/tumblr.com/tumblr.api.js
+++ b/plugins/domains/tumblr.com/tumblr.api.js
@@ -1,6 +1,5 @@
 import cheerio_pkg from 'cheerio';
 const $ = cheerio_pkg.default;
-import * as _ from 'underscore';
 
 export default {
 
@@ -24,7 +23,7 @@ export default {
             author: tumblr_post.blog_name,
             author_url: 'https://' + tumblr_post.blog_name + '.tumblr.com',
             canonical: tumblr_post.permalink_url || tumblr_post.post_url,
-            tags: _.unique([].concat(tumblr_post.tags, tumblr_post.featured_in_tag || [])).join(', '),
+            tags: tumblr_post.tags && tumblr_post.tags.join(', '),
             shortlink: tumblr_post.short_url,
             date: tumblr_post.date,
             duration: tumblr_post.duration,
@@ -81,7 +80,7 @@ export default {
             prepareResult: function (error, response, body, cb) {
 
                 if (error || body.errors) {
-                    return cb(error || 'There was a Tumblr API error error');
+                    return cb(error || body.errors || 'There was a Tumblr API error');
                 }
 
                 if (!body.meta) {
diff --git a/plugins/links/hosted/23video-hosted.js b/plugins/links/hosted/23video-hosted.js
index f07fda3a1..9ae4bb470 100644
--- a/plugins/links/hosted/23video-hosted.js
+++ b/plugins/links/hosted/23video-hosted.js
@@ -1,48 +1,18 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
-
 export default {
 
+    provides: ["oembed_domain"],
+
     // linked to oembed, so won't run for all URLs
-    getLink: function(meta, oembed, whitelistRecord) {
+    getData: function(meta, oembed, iframe, whitelistRecord) {
 
         if (whitelistRecord.isDefault
-            && oembed.type == 'video' && oembed.width && oembed.height
-            && ((meta.em && meta.em.schema == '23video') || oembed.provider_name == 'TwentyThree')) {
-
-            var players = [];
+            && oembed.type == 'video'
+            && /\/v\.ihtml(?:\/player\.html)?/i.test(iframe.src)
+            && ((meta.em && meta.em.schema == '23video') || /^twentythree$/i.test(oembed.provider_name))) {
 
-            if (meta.video_src
-                && /https?:\/\/[^.]+\.23video\.com\/\d+\/\d+\/\w+\/[^\?]+\.mp4/i.test(meta.video_src)) {
-
-                players.push({
-                    href: meta.video_src,
-                    type: CONFIG.T.video_mp4,
-                    rel: [CONFIG.R.player, CONFIG.R.html5],
-                    'aspect-ratio': oembed.width / oembed.height
-                });
+            return {
+                oembed_domain: "23video.com"
             }
-
-            var $container = cheerio('<div>');
-            try {
-                $container.html(oembed.html);
-            } catch (ex) {}
-
-            var $iframe = $container.find('iframe');
-
-            if ($iframe.length == 1 && /\/v\.ihtml(?:\/player\.html)?/i.test($iframe.attr('src'))) {
-
-                players.push({
-                    href: $iframe.attr('src'),                    
-                    type: CONFIG.T.text_html,
-                    rel: [CONFIG.R.player, CONFIG.R.html5],
-                    'aspect-ratio': oembed.width / oembed.height,
-                    autoplay: 'autoPlay=1'
-                });
-
-            }
-
-            return players;
         }
     },
 
@@ -52,7 +22,7 @@ export default {
         "https://video.twentythree.net/intro-to-twentythrees-player-builder",
         "http://videos.theconference.se/paul-adams-solving-real-world-problems",
         "http://www.fftv.no/skipatruljen-s3e3-voss-resort",
-        "https://videos.23video.com/novo-nordisk",
+        // "https://videos.23video.com/novo-nordisk",
         "http://video.nextconf.eu/video/1880845/data-without-limits",
         "http://stream.umbraco.org/v.ihtml?source=share&photo%5fid=11665495&autoPlay=0"
     ]
diff --git a/plugins/links/oembed-photo-html.js b/plugins/links/oembed-photo-html.js
index 5f0907d49..113099be3 100644
--- a/plugins/links/oembed-photo-html.js
+++ b/plugins/links/oembed-photo-html.js
@@ -1,6 +1,3 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
-
 export default {
 
     // this is the case of oembed photo or image, but with the html field
@@ -20,17 +17,11 @@ export default {
             };
 
 
-            var $container = cheerio('<div>');
-            try {
-                $container.html(oembed.html5 || oembed.html);
-            } catch (ex) {}
-
-            var $iframe = $container.find('iframe');
-
+            var iframe = oembed.getIframe();
 
             // if embed code contains <iframe>, return src
-            if ($iframe.length == 1) {
-                image.href = $iframe.attr('src');
+            if (iframe.src) {
+                image.href = iframe.src;
             } else { 
                 image.html = oembed.html || oembed.html5; // will render in an iframe
             }
diff --git a/plugins/links/oembed-video.js b/plugins/links/oembed-video.js
index bf96b981c..3c22ac507 100644
--- a/plugins/links/oembed-video.js
+++ b/plugins/links/oembed-video.js
@@ -14,25 +14,11 @@ export default {
             rel:[CONFIG.R.oembed, CONFIG.R.player]
         };
 
-
-        // allow encoded entities if they start from $lt;
-        // ex.: http://www.nfb.ca/film/wild_life/
-        var html = oembed.html5 || oembed.html; 
-        if (/^&lt;/i.test(html)) {
-            html = entities.decodeHTML(html);
-        }
-
-        var $container = cheerio('<div>');
-        try {
-            $container.html(html);
-        } catch (ex) {}
-
-        var $iframe = $container.find('iframe');
-
+        var iframe = oembed.getIframe();
 
         // if embed code contains <iframe>, return src
-        if ($iframe.length == 1 && $iframe.attr('src')) {
-            player.href = $iframe.attr('src');
+        if (iframe && iframe.src) {
+            player.href = iframe.src;
 
             if (whitelistRecord.isAllowed('oembed.video', 'ssl')) {
                 player.href = player.href.replace(/^http:\/\//i, '//');
@@ -52,7 +38,7 @@ export default {
             }
 
         } else { 
-            player.html = html; // will render in an iframe
+            player.html = oembed.html; // will render in an iframe
             player.type = CONFIG.T.text_html;
         }
 
@@ -76,8 +62,8 @@ export default {
             player.rel.push(CONFIG.R.html5);
         }
 
-        if ($iframe.length == 1 && $iframe.attr('allow')) {
-            player.rel = player.rel.concat($iframe.attr('allow').replace(/autoplay;?\s?\*?/ig, '').split(/\s?\*?;\s?\*?/g));
+        if (iframe && iframe.allow) {
+            player.rel = player.rel.concat(iframe.allow.replace(/autoplay;?\s?\*?/ig, '').split(/\s?\*?;\s?\*?/g));
         }        
 
         return player;
@@ -98,7 +84,6 @@ export default {
     /*
     tests: [
         "http://sports.pixnet.net/album/video/183041064", 
-        "http://video.yandex.ua/users/enema-bandit/view/11/?ncrnd=4917#hq"
     ]
     */
 };
\ No newline at end of file
diff --git a/plugins/meta/ld-article.js b/plugins/meta/ld-article.js
index 1a994145e..228c906dd 100644
--- a/plugins/meta/ld-article.js
+++ b/plugins/meta/ld-article.js
@@ -15,8 +15,8 @@ export default {
                 return $container.text();
             }
         }
-        if (ld.newsarticle) {
 
+        if (ld.newsarticle) {
             return {
                 title: clean(ld.newsarticle.headline),
                 category: clean(ld.newsarticle.articlesection),

From 7b081427aa60a72c6879ffedc5e54b2bdf692087 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 20 Oct 2021 18:12:06 +0300
Subject: [PATCH 038/102] fix sax import

---
 lib/plugins/system/oembed/oembedUtils.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index 9078a2897..47e9c3bef 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -1,4 +1,4 @@
-import * as sax from 'sax';
+import sax from 'sax';
 import * as urlLib from 'url';
 import * as async from 'async';
 import * as utils from '../../../utils.js';

From 17cb1d50ba0eb0e524a0d3b40709fd519b3cf136 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 20 Oct 2021 21:42:31 +0300
Subject: [PATCH 039/102] use `helix-fetch`

---
 lib/fetch.js |  20 +++++++++
 lib/utils.js |  98 +++++++++++++------------------------------
 package.json |   2 +-
 yarn.lock    | 114 ++++++++++++++++-----------------------------------
 4 files changed, 86 insertions(+), 148 deletions(-)
 create mode 100644 lib/fetch.js

diff --git a/lib/fetch.js b/lib/fetch.js
new file mode 100644
index 000000000..6547482b7
--- /dev/null
+++ b/lib/fetch.js
@@ -0,0 +1,20 @@
+import { fetch, AbortController } from '@adobe/helix-fetch';
+
+export function fetchStream(options) {
+    const createAbortController = options.createAbortController;
+    const abortController = createAbortController ? new AbortController() : null;
+    const fetch_options = Object.assign({}, options, {
+        signal: abortController && abortController.signal
+    });
+    return new Promise((resolve, reject) => {
+        fetch(options.uri, fetch_options)
+            .then(response => {
+                var stream = response.body;
+                stream.status = response.status;
+                stream.headers = response.headers.plain();
+                stream.abortController = abortController;
+                resolve(stream);
+            })
+            .catch(reject);
+    });
+};
\ No newline at end of file
diff --git a/lib/utils.js b/lib/utils.js
index 83cde0483..75b623ee9 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,7 +1,6 @@
 import * as events from 'events';
 import request from 'request';
-import { context, AbortController } from 'fetch-h2';
-const { fetch } = context( );
+import { fetchStream } from './fetch.js';
 import * as iconv from 'iconv-lite';
 import * as async from 'async';
 import imagesize from 'probe-image-size';
@@ -70,22 +69,23 @@ export function getMaxCacheTTL(url, options, default_min_ttl) {
 
 export function prepareRequestOptions(request_options, options) {
 
-    var url = request_options.uri || request_options.url;
+    if (request_options.url && !request_options.uri) {
+        request_options.uri = request_options.url;
+    }
+    var uri = request_options.uri;
+    delete request_options.url;
 
     var disableHttp2 = options && options.disableHttp2 || CONFIG.DISABLE_HTTP2;
 
     if (CONFIG.PROXY || (options && options.proxy)) {
 
         var proxy = (options && options.proxy) || CONFIG.PROXY.find(p => {
-            return p && p.re && p.re.some(re => url.match(re));
+            return p && p.re && p.re.some(re => uri.match(re));
         });
         if (proxy) {
             if (proxy.prerender && CONFIG.PRERENDER_URL) {
-                delete request_options.uri;
-                delete request_options.url;
                 // `url` used above for agent check.
-                url = CONFIG.PRERENDER_URL + encodeURIComponent(url);
-                request_options.uri = url;
+                request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
             }
             if (proxy.proxy_server) {
                 request_options.proxy = proxy.proxy_server;
@@ -109,7 +109,7 @@ export function prepareRequestOptions(request_options, options) {
 
     if (!disableHttp2
         && !request_options.proxy
-        && /^https:/.test(url)
+        && /^https:/.test(uri)
         && (!request_options.method
             || request_options.method === 'GET'
             || request_options.method === 'HEAD')) {
@@ -125,7 +125,6 @@ export function prepareRequestOptions(request_options, options) {
         request_options.headers['Accept-Language'] = lang.replace('_', '-') + CONFIG.ACCEPT_LANGUAGE_SUFFIX;
     }
 
-    prepareEncodedUri(request_options, 'url');
     prepareEncodedUri(request_options, 'uri');
 
     return request_options;
@@ -152,10 +151,9 @@ export function getUrlFunctional(url, options, callbacks) {
 
             callbacks.onError && callbacks.onError(error);
         })
-        .on('abortController', function(abortController) {
-            callbacks.onAbortController && callbacks.onAbortController(abortController);
-        })
         .on('response', function(response) {
+            // TODO: remove abort event
+            callbacks.onAbortController && callbacks.onAbortController(response.abortController);
             callbacks.onResponse && callbacks.onResponse(response);
         });
 }
@@ -172,8 +170,6 @@ export function getUrlFunctional(url, options, callbacks) {
  */
  export function getUrl(url, options) {
 
-    const controller = new AbortController();
-
     var req = new events.EventEmitter();
 
     var options = options || {};
@@ -184,8 +180,6 @@ export function getUrlFunctional(url, options, callbacks) {
         jar = request.jar();
     }
 
-    var response;
-
     process.nextTick(function() {
         try {
 
@@ -197,41 +191,28 @@ export function getUrlFunctional(url, options, callbacks) {
                     'User-Agent': options.user_agent || CONFIG.USER_AGENT,
                     //'Connection': 'keep-alive',                    
                     'Accept': '*/*'
-                    // gzip, br, etc - set by `node-fetch` `compress: true` option.
+                    // gzip, br, etc - set by `node-fetch` default `compress: true` option.
                 },
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
                 jar: jar,
 
                 // New for `node-fetch`
-                signal: controller.signal,
-                compress: true,
                 redirect: options.followRedirect ? 'follow' : 'manual',
-                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
+                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
+                createAbortController: true
             }, options);
 
-            fetch(request_options.url || request_options.uri, request_options)
-                .then(res => {
-                    response = res;
-                    return res.readable();
-                })
+            fetchStream(request_options)
                 .then(stream => {
                     if (!options.asBuffer) {
                         stream.setEncoding("binary");
                     }
-
-                    stream.status = response.status;
-                    stream.headers = response.headers.toJSON();
-
-                    // TODO:
                     req.emit('response', stream);
                 })
                 .catch(error => {
                     req.emit('error', error, {http2Agent: false/*request_options.agent === http2Agent*/});
                 });
 
-
-            req.emit('abortController', controller);
-
         } catch (ex) {
             console.error('Error on getUrl for', url, '.\n Error:' + ex);
             req.emit('error', ex);
@@ -241,7 +222,6 @@ export function getUrlFunctional(url, options, callbacks) {
 };
 
 export function getHeadFunctional(url, options, callbacks) {
-
     getHead(url, options)
         .on('error', function(error) {
 
@@ -258,9 +238,6 @@ export function getHeadFunctional(url, options, callbacks) {
 
             callbacks.onError && callbacks.onError(error);
         })
-        .on('abortController', function(abortController) {
-            callbacks.onAbortController && callbacks.onAbortController(abortController);
-        })
         .on('response', function(response) {
             callbacks.onResponse && callbacks.onResponse(response);
         });
@@ -268,8 +245,6 @@ export function getHeadFunctional(url, options, callbacks) {
 
 var getHead = function(url, options) {
 
-    const controller = new AbortController();
-
     var req = new events.EventEmitter();
 
     var options = options || {};
@@ -280,8 +255,6 @@ var getHead = function(url, options) {
         jar = request.jar();
     }
 
-    var response;
-
     process.nextTick(function() {
         try {
 
@@ -301,32 +274,21 @@ var getHead = function(url, options) {
                 },
 
                 // New for `node-fetch`.
-                signal: controller.signal,
-                compress: true,
                 redirect: options.followRedirect ? 'follow' : 'manual',
                 follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
+                // No abort controller for head.
             }, {
                 disableHttp2: options && options.disableHttp2
             });
 
-            fetch(request_options.url || request_options.uri, request_options)
-                .then(res => {
-                    response = res;
-                    return res.readable();
-                })
-                .then(stream => {
-
-                    stream.status = response.status;
-                    stream.headers = response.headers.toJSON();
-
-                    req.emit('response', stream);
+            fetchStream(request_options)
+                .then(response => {
+                    req.emit('response', response);
                 })
                 .catch(error => {
                     req.emit('error', error);
                 });
 
-            req.emit('abortController', controller);
-
         } catch (ex) {
             console.error('Error on getHead for', url, '.\n Error:' + ex);
             req.emit('error', ex);
@@ -934,7 +896,9 @@ export function sendLogToWhitelist(uri, context) {
             data.oembed = oembedHref;
         }
 
-        fetch(CONFIG.WHITELIST_LOG_URL, {
+        // TODO: check options
+        fetchStream({
+            uri: CONFIG.WHITELIST_LOG_URL,
             method: 'GET',
             qs: data
         })
@@ -1102,8 +1066,6 @@ export function generateLinksHtml(data, options) {
 
 var getUriStatusPrivate = function(uri, options, cb) {
 
-    const controller = new AbortController();
-
     var request_options = prepareRequestOptions({
         uri: uri,
         method: 'GET',
@@ -1114,26 +1076,24 @@ var getUriStatusPrivate = function(uri, options, cb) {
         jar: request.jar(),   //Enable cookies, uses new jar
 
         // New by 'node-fetch'.
-        signal: controller.signal,
-        compress: true,
         redirect: 'follow', // Default value.
-        follow: CONFIG.MAX_REDIRECTS
+        follow: CONFIG.MAX_REDIRECTS,
+        createAbortController: true
     }, {
         disableHttp2: options.disableHttp2
     })
 
     try {
-        fetch(request_options.uri || request_options.url, request_options)
+        fetchStream(request_options)
             .then(res => {
-                controller.abort();
-
+                res.abortController.abort();
                 var data = {
                     code: res.status,
-                    content_type: res.headers && res.headers.get('content-type'),
-                    content_length: res.headers && res.headers.get('content-length') ? parseInt(res.headers.get('content-length') || '0', 10) : null
+                    content_type: res.headers && res.headers['content-type'],
+                    content_length: res.headers && res.headers['content-length'] ? parseInt(res.headers['content-length'] || '0', 10) : null
                 };
                 if (options.checkHeaders) {
-                    data.headers = res.headers.toJSON();
+                    data.headers = res.headers;
                 }
                 cb(null, data);
             })
diff --git a/package.json b/package.json
index 651616a08..39eca5bce 100644
--- a/package.json
+++ b/package.json
@@ -20,13 +20,13 @@
     },
     "license": "MIT",
     "dependencies": {
+        "@adobe/helix-fetch": "^3.0.0",
         "async": "2.4.1",
         "cheerio": "^1.0.0-rc.10",
         "chokidar": "^3.3.1",
         "ejs": "^3.1.6",
         "entities": "1.1.1",
         "express": "^4.16.3",
-        "fetch-h2": "^3.0.1",
         "globby": "^12.0.2",
         "graceful-cluster": "0.0.3",
         "htmlparser2": "^7.1.2",
diff --git a/yarn.lock b/yarn.lock
index db4311449..e1a905870 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,6 +2,15 @@
 # yarn lockfile v1
 
 
+"@adobe/helix-fetch@^3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@adobe/helix-fetch/-/helix-fetch-3.0.0.tgz#89c9c01ca3894375d6ca6cefa2ce974bcbc37f46"
+  integrity sha512-yvkOBwXLYchH/ql/KZ0Rp7QLuw0J38q0ZgOiPeF4ZPrBFfyJD6HmAX5AjvHEaofOyPKIJmlZb5KD+6kjE+01kg==
+  dependencies:
+    debug "4.3.2"
+    http-cache-semantics "4.1.0"
+    lru-cache "6.0.0"
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -50,11 +59,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.1.tgz#2e50a649a50fc403433a14f829eface1a3443e97"
   integrity sha512-PYGcJHL9mwl1Ek3PLiYgyEKtwTMmkMw4vbiyz/ps3pfdRYLVv+SN7qHVAImrjdAXxgluDEw6Ph4lyv+m9UpRmA==
 
-"@types/tough-cookie@^4.0.0":
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.1.tgz#8f80dd965ad81f3e1bc26d6f5c727e132721ff40"
-  integrity sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==
-
 abbrev@1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
@@ -83,11 +87,6 @@ ajv@^6.12.3:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-already@^2.2.1:
-  version "2.2.1"
-  resolved "https://registry.yarnpkg.com/already/-/already-2.2.1.tgz#98c257baa0d3fe62d85163ff288235ba11e3f188"
-  integrity sha512-qk6RIVMS/R1yTvBzfIL1T76PsIL7DIVCINoLuFw2YXKLpLtsTobqdChMs8m3OhuPS3CEE3+Ra5ibYiqdyogbsQ==
-
 ansi-styles@^3.2.1:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d"
@@ -268,11 +267,6 @@ bytes@3.1.0:
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
   integrity sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==
 
-callguard@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/callguard/-/callguard-2.0.0.tgz#32f98348ff82cb1dfcf7d1b198108cf4f5b64c1f"
-  integrity sha512-I3nd+fuj20FK1qu00ImrbH+II+8ULS6ioYr9igqR1xyqySoqc3DiHEyUM0mkoAdKeLGg2CtGnO8R3VRQX5krpQ==
-
 caseless@~0.12.0:
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
@@ -463,6 +457,13 @@ debug@3.1.0:
   dependencies:
     ms "2.0.0"
 
+debug@4.3.2:
+  version "4.3.2"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"
+  integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==
+  dependencies:
+    ms "2.1.2"
+
 debug@^3.1.0:
   version "3.2.7"
   resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a"
@@ -726,19 +727,6 @@ feedparser@2.2.0:
     readable-stream "^2.2.2"
     sax "^1.2.1"
 
-fetch-h2@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/fetch-h2/-/fetch-h2-3.0.1.tgz#e49ab81416ab90cf815c27584c6b073e584c928e"
-  integrity sha512-d7xOZHI4Wd31XJe9ib4MmncauFeojN5WOedOwDberYVysH0jcicu7WimZfsrEwzCV/EJPCSDwwwDTRWlCP8QtQ==
-  dependencies:
-    "@types/tough-cookie" "^4.0.0"
-    already "^2.2.1"
-    callguard "^2.0.0"
-    get-stream "^6.0.1"
-    through2 "^4.0.2"
-    to-arraybuffer "^1.0.1"
-    tough-cookie "^4.0.0"
-
 filelist@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/filelist/-/filelist-1.0.2.tgz#80202f21462d4d1c2e214119b1807c1bc0380e5b"
@@ -805,11 +793,6 @@ fsevents@~2.3.2:
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
 
-get-stream@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7"
-  integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==
-
 getpass@^0.1.1:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa"
@@ -931,6 +914,11 @@ htmlparser2@^7.1.2:
     domutils "^2.8.0"
     entities "^3.0.1"
 
+http-cache-semantics@4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390"
+  integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==
+
 http-errors@1.7.2, http-errors@~1.7.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.2.tgz#4f5029cf13239f31036e5b2e55292bcfbcc85c8f"
@@ -1153,6 +1141,13 @@ lodash@^4.14.0:
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
 
+lru-cache@6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94"
+  integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==
+  dependencies:
+    yallist "^4.0.0"
+
 media-typer@0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
@@ -1517,7 +1512,7 @@ proxy-addr@~2.0.5:
     forwarded "~0.1.2"
     ipaddr.js "1.9.1"
 
-psl@^1.1.28, psl@^1.1.33:
+psl@^1.1.28:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24"
   integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==
@@ -1582,15 +1577,6 @@ readable-stream@1.0:
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-stream@3:
-  version "3.6.0"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
-  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
-  dependencies:
-    inherits "^2.0.3"
-    string_decoder "^1.1.1"
-    util-deprecate "^1.0.1"
-
 readable-stream@^2.2.2, readable-stream@^2.3.5:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@@ -1737,7 +1723,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
-safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0:
+safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@@ -1886,13 +1872,6 @@ stream-parser@~0.3.1:
   dependencies:
     debug "2"
 
-string_decoder@^1.1.1:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
-  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
-  dependencies:
-    safe-buffer "~5.2.0"
-
 string_decoder@~0.10.x:
   version "0.10.31"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
@@ -1943,18 +1922,6 @@ supports-color@^5.3.0:
   dependencies:
     has-flag "^3.0.0"
 
-through2@^4.0.2:
-  version "4.0.2"
-  resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
-  integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
-  dependencies:
-    readable-stream "3"
-
-to-arraybuffer@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43"
-  integrity sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=
-
 to-regex-range@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
@@ -1967,15 +1934,6 @@ toidentifier@1.0.0:
   resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553"
   integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==
 
-tough-cookie@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4"
-  integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==
-  dependencies:
-    psl "^1.1.33"
-    punycode "^2.1.1"
-    universalify "^0.1.2"
-
 tough-cookie@~2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
@@ -2031,11 +1989,6 @@ underscore@*, underscore@^1.13.1, underscore@^1.8.3:
   resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.1.tgz#0c1c6bd2df54b6b69f2314066d65b6cde6fcf9d1"
   integrity sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==
 
-universalify@^0.1.2:
-  version "0.1.2"
-  resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
-  integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
-
 unpipe@1.0.0, unpipe@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
@@ -2048,7 +2001,7 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
-util-deprecate@^1.0.1, util-deprecate@~1.0.1:
+util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
@@ -2081,3 +2034,8 @@ wrappy@1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
   integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
+
+yallist@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
+  integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==

From 26e53dccc080530577e9bde1f6980e9230bf10ac Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 20 Oct 2021 23:07:12 +0300
Subject: [PATCH 040/102] fetch: disable cache

---
 lib/fetch.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 6547482b7..8d2f8e7fe 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,4 +1,6 @@
-import { fetch, AbortController } from '@adobe/helix-fetch';
+import { noCache, AbortController } from '@adobe/helix-fetch';
+
+const { fetch } = noCache();
 
 export function fetchStream(options) {
     const createAbortController = options.createAbortController;

From 0f9deed4d0bfc1bd1f0f594d86a689cf59596dc9 Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Thu, 21 Oct 2021 09:20:56 -0400
Subject: [PATCH 041/102] Get rid of some of the underscores

---
 lib/cache-engines/memcached.js             | 184 ++++++++++-----------
 lib/cache.js                               |   6 +-
 lib/core.js                                |  12 +-
 lib/loader/pluginLoader.js                 |   9 +-
 lib/plugins/system/meta/HTMLMetaHandler.js |   5 +-
 lib/utils.js                               |   2 +-
 lib/whitelist.js                           |   2 +-
 7 files changed, 107 insertions(+), 113 deletions(-)

diff --git a/lib/cache-engines/memcached.js b/lib/cache-engines/memcached.js
index f9c5ab9e3..6a30f244d 100644
--- a/lib/cache-engines/memcached.js
+++ b/lib/cache-engines/memcached.js
@@ -1,134 +1,132 @@
+import log from '../../logging.js';
+import * as crypto from 'crypto';
+import * as Memcached from 'memcached';
 
-    import log from '../../logging.js';
-    import * as crypto from 'crypto';
-    import * as Memcached from 'memcached';
-    import * as _ from 'underscore';
+var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options);
 
-    var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options);
+var timeout = CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options && CONFIG.MEMCACHED_OPTIONS.options.timeout;
 
-    var timeout = CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options && CONFIG.MEMCACHED_OPTIONS.options.timeout;
+var MEMCACHED_MAX_DATA_SIZE = 1024 * 1024;  // Megabyte.
 
-    var MEMCACHED_MAX_DATA_SIZE = 1024 * 1024;  // Megabyte.
+function safeKey(key) {
+    return crypto.createHash('md5').update(key).digest("hex");
+}
 
-    function safeKey(key) {
-        return crypto.createHash('md5').update(key).digest("hex");
-    }
+function _findKeyMeta(k) {
 
-    function _findKeyMeta(k) {
 
+    var sk = safeKey(k);
 
-        var sk = safeKey(k);
+    memcached.items(function(a, b){
 
-        memcached.items(function(a, b){
+        var stubs = Object.keys(b[0]);
 
-            var stubs = _.keys(b[0]);
+        stubs.forEach(function(sid) {
 
-            stubs.forEach(function(sid) {
+            var servers = CONFIG.MEMCACHED_OPTIONS.locations;
+            if (servers instanceof Object) {
+                servers = Object.keys(servers);
+            }
+            if (typeof servers === 'string') {
+                servers = [servers];
+            }
 
-                var servers = CONFIG.MEMCACHED_OPTIONS.locations;
-                if (servers instanceof Object) {
-                    servers = _.keys(servers);
-                }
-                if (typeof servers === 'string') {
-                    servers = [servers];
-                }
+            servers.forEach(function(s) {
+                memcached.cachedump(s, parseInt(sid), 0, function(a, b) {
 
-                servers.forEach(function(s) {
-                    memcached.cachedump(s, parseInt(sid), 0, function(a, b) {
+                    if (!(b instanceof Array)) {
+                        b = [b];
+                    }
 
-                        if (!(b instanceof Array)) {
-                            b = [b];
+                    b.forEach(function(k) {
+                        if (k.key === sk) {
+                            console.log(' - key', k);
+                            console.log(' - now is', new Date());
+                            console.log(' - exp in', new Date(k.s * 1000));
                         }
-
-                        b.forEach(function(k) {
-                            if (k.key === sk) {
-                                console.log(' - key', k);
-                                console.log(' - now is', new Date());
-                                console.log(' - exp in', new Date(k.s * 1000));
-                            }
-                        })
-                    });
+                    })
                 });
             });
         });
-    }
-
-    export function set(_key, data, options) {
-
-        var key = (!options || !options.raw) ? safeKey(_key) : _key;
+    });
+}
 
-        //console.log(key, (!options || !options.raw) ? JSON.stringify(data) : data);
+export function set(_key, data, options) {
 
-        // Warning: value and lifetime argumets switched, bug in docs.
-        // Warning: need replace /n if raw saved. Memcached in nginx read bug.
-        var storedData = (!options || !options.raw) ? JSON.stringify(data) : data.replace(/\n/g, '');
+    var key = (!options || !options.raw) ? safeKey(_key) : _key;
 
-        if (storedData && storedData.length > MEMCACHED_MAX_DATA_SIZE) {
+    //console.log(key, (!options || !options.raw) ? JSON.stringify(data) : data);
 
-            log('   -- Memcached handled set error ' + _key + ': The length of the value is ' + storedData.length + ' and greater than ' + MEMCACHED_MAX_DATA_SIZE);
+    // Warning: value and lifetime argumets switched, bug in docs.
+    // Warning: need replace /n if raw saved. Memcached in nginx read bug.
+    var storedData = (!options || !options.raw) ? JSON.stringify(data) : data.replace(/\n/g, '');
 
-        } else {
+    if (storedData && storedData.length > MEMCACHED_MAX_DATA_SIZE) {
 
-            memcached.set(key, storedData, options && options.ttl || CONFIG.CACHE_TTL, function(error){
-                if (error) {
-                    log('   -- Memcached set error ' + _key + ' ' + error);
-                }
-            });
-        }
-    };
+        log('   -- Memcached handled set error ' + _key + ': The length of the value is ' + storedData.length + ' and greater than ' + MEMCACHED_MAX_DATA_SIZE);
 
-    export function get(_key, cb) {
+    } else {
 
-        var key = safeKey(_key);
+        memcached.set(key, storedData, options && options.ttl || CONFIG.CACHE_TTL, function(error){
+            if (error) {
+                log('   -- Memcached set error ' + _key + ' ' + error);
+            }
+        });
+    }
+};
 
-        var timeoutId, finished = false;
+export function get(_key, cb) {
 
-        if (timeout) {
-            setTimeout(function() {
-                if (finished) {
-                    return;
-                }
-                finished = true;
-                log('   -- Memcached soft timeout on get ' + _key);
-                cb(null, null);
-            }, timeout);
-        }
+    var key = safeKey(_key);
 
-        memcached.get(key, function (error, data) {
+    var timeoutId, finished = false;
 
+    if (timeout) {
+        setTimeout(function() {
             if (finished) {
                 return;
             }
             finished = true;
+            log('   -- Memcached soft timeout on get ' + _key);
+            cb(null, null);
+        }, timeout);
+    }
 
-            clearTimeout(timeoutId);
+    memcached.get(key, function (error, data) {
 
-            if (error) {
-                log('   -- Memcached get error ' + _key + ' ' + error);
-                // Fail silent.
-                return cb(null, null);
-            }
+        if (finished) {
+            return;
+        }
+        finished = true;
 
-            if (typeof data !== 'string') {
-                return cb(null, data);
-            }
+        clearTimeout(timeoutId);
 
-            try {
-                var parsedData = JSON.parse(data);
-            } catch(ex) {
-                log('   -- Memcached: error JSON parse value ' + _key + ' ' + ex.message);
-                return cb(null, null);
-            }
+        if (error) {
+            log('   -- Memcached get error ' + _key + ' ' + error);
+            // Fail silent.
+            return cb(null, null);
+        }
 
-            if (parsedData && typeof parsedData === 'object' && _key in parsedData) {
-                // Support old multi cached data.
-                parsedData = parsedData[_key];
-            }
+        if (typeof data !== 'string') {
+            return cb(null, data);
+        }
 
-            cb(null, parsedData);
-        });
-    };
+        try {
+            var parsedData = JSON.parse(data);
+        } catch(ex) {
+            log('   -- Memcached: error JSON parse value ' + _key + ' ' + ex.message);
+            return cb(null, null);
+        }
+
+        if (parsedData && typeof parsedData === 'object' && _key in parsedData) {
+            // Support old multi cached data.
+            parsedData = parsedData[_key];
+        }
+
+        cb(null, parsedData);
+    });
+};
 
-    export function getClient() {
-        return memcached;
-    };
+export function getClient() {
+    return memcached;
+};
diff --git a/lib/cache.js b/lib/cache.js
index b09ef18ef..26ac19332 100644
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -185,9 +185,9 @@
 
         var that = this;
 
-        var getsCount = _.keys(this.getCbDict).length;
+        var getsCount = Object.keys(this.getCbDict).length;
 
-        var keysList = _.keys(this.keysDict).sort();
+        var keysList = Object.keys(this.keysDict).sort();
         this.multiKey = keysList.join(':');
         if (keysList.length > 1) {
             this.multiKey = 'multi:' + this.multiKey;
@@ -263,7 +263,7 @@
     };
 
     MultiCache.prototype._checkAllSet = function() {
-        if (_.keys(this.keysDict).length === _.keys(this.setKeysDict).length && this.multiKey) {
+        if (Object.keys(this.keysDict).length === Object.keys(this.setKeysDict).length && this.multiKey) {
             this._runAllSets();
         }
     };
diff --git a/lib/core.js b/lib/core.js
index 90df33ce4..86fe10f76 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -32,7 +32,7 @@
         *
         *
         * loadedParams - list of currently loaded params keys.
-        * var loadedParams = _.keys(context);
+        * var loadedParams = Object.keys(context);
         * loadedParams = ['cb', 'uri', 'title', 'meta']
         *
         *
@@ -79,7 +79,7 @@
         * mandatoryParams - list of new params not used by plugins. Core will find what can use them.
         * 'mandatoryParams' enables mandatory mode: function will use _only_ methods which has this input 'mandatoryParams'.
         * This is used for "go down by tree" algorithm.
-        * var mandatoryParams = _.difference(loadedParams, _.keys(usedParams));
+        * var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams));
         * mandatoryParams = [
         *   paramName
         * ]
@@ -346,9 +346,9 @@
     * */
     function runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb) {
 
-        var loadedParams = _.keys(context);
+        var loadedParams = Object.keys(context);
 
-        var mandatoryParams = _.difference(loadedParams, _.keys(usedParams));
+        var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams));
 
         // Reset scanned plugins for each iteration.
         var scannedPluginsIds = {};
@@ -1345,7 +1345,7 @@
 
             var value = searchParamInObj(0, bits, options.providerOptions);
 
-            if (_.isArray(value)) {
+            if (Array.isArray(value)) {
 
                 // ok.
 
@@ -1648,7 +1648,7 @@
                             delete options.skipFallbackNotification;
                         } else {
                             fallbackInfo = {
-                                usedDomainsList: _.keys(usedDomains),
+                                usedDomainsList: Object.keys(usedDomains),
                                 uri: uri
                             };
                         }
diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js
index 0864b0bf7..edbc3274d 100644
--- a/lib/loader/pluginLoader.js
+++ b/lib/loader/pluginLoader.js
@@ -1,7 +1,6 @@
 
     import * as fs from 'fs';
     import * as path from 'path';
-    import * as _ from 'underscore';
     import * as node_jslint from 'jslint';
     import { readFile } from 'fs/promises';
     var JSLINT = node_jslint.load('latest');
@@ -174,9 +173,7 @@
             }
 
             // Check if have required method.
-            var hasSign = _.some(PLUGINS_FIELDS, function(sign) {
-                return sign in plugin;
-            });
+            var hasSign = PLUGINS_FIELDS.some(sign => sign in plugin);
             if (!hasSign) {
                 console.warn("No plugin methods in " + pluginPath + ". Add exports.notPlugin = true; to skip this warning.");
                 return;
@@ -225,7 +222,7 @@
                     pluginDeclaration.re = [plugin.re];
                 } else if (plugin.re instanceof Array) {
 
-                    if (!_.every(plugin.re, function(re) { return re instanceof RegExp; })) {
+                    if (!plugin.re.every(re => re instanceof RegExp)) {
                         console.error('Invalid RegExp in re of', pluginPath);
                         return;
                     }
@@ -561,7 +558,7 @@
             }
         });
 
-        var keys = _.keys(metaMappingsNotSorted);
+        var keys = Object.keys(metaMappingsNotSorted);
         keys.sort();
         keys.forEach(function(key) {
             metaMappings[key] = metaMappingsNotSorted[key];
diff --git a/lib/plugins/system/meta/HTMLMetaHandler.js b/lib/plugins/system/meta/HTMLMetaHandler.js
index 590ea2bc9..2f86811fe 100644
--- a/lib/plugins/system/meta/HTMLMetaHandler.js
+++ b/lib/plugins/system/meta/HTMLMetaHandler.js
@@ -1,5 +1,4 @@
 import { decodeHTML5 } from 'entities';
-import * as _ from 'underscore';
 import * as url from 'url';
 import * as utils from '../../../utils.js';
 import { ldParser } from './ld-json.js';
@@ -323,7 +322,7 @@ HTMLMetaHandler.prototype._finalMerge = function(tag) {
     function getSingleValue(parentName, obj) {
         var key = getDefaultKey(parentName);
         if (key in obj) {
-            if (_.keys(obj).length === 1) {
+            if (Object.keys(obj).length === 1) {
                 return obj[key];
             }
         }
@@ -515,7 +514,7 @@ function merge(parentObj, props, value) {
         // New node.
         parentNode[currentNodeName] = value;
 
-    } else if (_.isArray(parentNode[currentNodeName])) {
+    } else if (Array.isArray(parentNode[currentNodeName])) {
 
         var append = false;
 
diff --git a/lib/utils.js b/lib/utils.js
index 75b623ee9..8a53e41aa 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -769,7 +769,7 @@ var minDate = new Date(1990, 1);
 
 export function unifyDate(date) {
 
-    if (_.isArray(date)) {
+    if (Array.isArray(date)) {
         date = date[0];
     }
 
diff --git a/lib/whitelist.js b/lib/whitelist.js
index 4783f3d60..eb738b239 100644
--- a/lib/whitelist.js
+++ b/lib/whitelist.js
@@ -235,7 +235,7 @@
 
         addWildcard();
 
-        log('Whitelist activated. Domains, including blacklisted:', _.keys(data.domains).length);
+        log('Whitelist activated. Domains, including blacklisted:', Object.keys(data.domains).length);
     }
 
     function readWhitelist(filename) {

From 76150eefe9a5bc49f2b3e78bf51017f69edbfda3 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 21 Oct 2021 20:23:44 +0300
Subject: [PATCH 042/102] request: implement using `helix-fetch` with most
 features

---
 lib/fetch.js   | 39 ++++++++++++++++++++++++++++++++++-----
 lib/request.js | 46 +++++++++++++++++++++++++++++++++++-----------
 lib/utils.js   | 48 ++++++++++++++++++++++++++----------------------
 3 files changed, 95 insertions(+), 38 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 8d2f8e7fe..b28145ce5 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,15 +1,17 @@
-import { noCache, AbortController } from '@adobe/helix-fetch';
+import { noCache, timeoutSignal, AbortController, createUrl } from '@adobe/helix-fetch';
 
 const { fetch } = noCache();
 
 export function fetchStream(options) {
     const createAbortController = options.createAbortController;
     const abortController = createAbortController ? new AbortController() : null;
-    const fetch_options = Object.assign({}, options, {
-        signal: abortController && abortController.signal
-    });
+    const fetch_options = Object.assign({}, options);
+    if (abortController) {
+        fetch_options.signal = abortController.signal;
+    }
+    const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
     return new Promise((resolve, reject) => {
-        fetch(options.uri, fetch_options)
+        fetch(uri, fetch_options)
             .then(response => {
                 var stream = response.body;
                 stream.status = response.status;
@@ -19,4 +21,31 @@ export function fetchStream(options) {
             })
             .catch(reject);
     });
+};
+
+export function fetchData(options) {
+    var json = options.json;
+    var response;
+    const fetch_options = Object.assign({}, options);
+    const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
+    return new Promise((resolve, reject) => {
+        fetch(uri, fetch_options)
+            .then(res => {
+                response = res;
+                json = json || (response.headers.get('content-type').indexOf('application/json') > -1);
+                if (json) {
+                    return response.json();
+                } else {
+                    return response.text();
+                }
+            })
+            .then(data => {
+                resolve({
+                    status: response.status,
+                    headers: response.headers.plain(),
+                    data: data
+                });
+            })
+            .catch(reject);
+    });
 };
\ No newline at end of file
diff --git a/lib/request.js b/lib/request.js
index 579c65a51..dd2cca0b6 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -1,10 +1,10 @@
-import request from 'request';
-import * as _ from 'underscore';
 import * as crypto from 'crypto';
 import { cache } from './cache.js';
 import * as utils from './utils.js';
 import log from '../logging.js';
 
+import { fetchData } from './fetch.js';
+
 var hash = function(value) {
     return '"' + crypto.createHash('md5').update(value).digest("hex") + '"';
 };
@@ -27,9 +27,24 @@ var hash = function(value) {
 * - callback - final callback will called with data received from `prepareResult`.
 *
 * */
+
+/*
+
+TOOD:
+
+options:
+
+jar: false,
+limit: 1, 
+timeout: 1000,
+
+oauth
+
+*/
+
 export default function(options, iframely_options, callback) {
 
-    options = _.extend({}, options);
+    options = Object.assign({}, options);
 
     if (typeof options.prepareResult !== 'function') {
         console.error('cached request call error: prepareResult not a function');
@@ -66,23 +81,26 @@ export default function(options, iframely_options, callback) {
 
     function doRealRequest(secondAttempt) {
 
-        request(utils.prepareRequestOptions(options, iframely_options), function(error, response, data) {
-
+        function finish(error, result) {
             if (error) {
-                var url = options.url || options.uri;
-
                 // Retry on ECONNRESET.
                 if (!secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                    log('   -- request.js ' + error.code + ' first', url);
+                    log('   -- request.js ' + error.code + ' first', options.uri);
                     process.nextTick(function() {
                         doRealRequest(true);
                     });
                     return;
                 } else if (secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                    log('   -- request.js ' + error.code + ' second', url);
+                    log('   -- request.js ' + error.code + ' second', options.uri);
                 }
             }
 
+            const response = result && {
+                statusCode: result.status,
+                headers: result.headers
+            };
+            const data = result && result.data;
+
             prepareResult(error, response, data, function(preparedError, preparedData) {
 
                 var doCaching;
@@ -121,7 +139,13 @@ export default function(options, iframely_options, callback) {
                 // Send prepared data up.
                 callback(preparedError, preparedData);
             });
-        });
+        }
+
+        fetchData(utils.prepareRequestOptions(options, iframely_options))
+            .then(result => {
+                finish(null, result);
+            })
+            .catch(finish);
     }
 
     if (refresh) {
@@ -156,7 +180,7 @@ export default function(options, iframely_options, callback) {
                 }
     
                 // Send cached data up.
-                prepareResult(null, _.extend(data.response, {fromRequestCache: true}), data.data, callback);
+                prepareResult(null, Object.assign(data.response, {fromRequestCache: true}), data.data, callback);
     
                 if (new_cache_key) {
                     cache.set('req:' + prefix + new_cache_key, data, {
diff --git a/lib/utils.js b/lib/utils.js
index 8a53e41aa..1f0ea47a9 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -185,6 +185,13 @@ export function getUrlFunctional(url, options, callbacks) {
 
             // TODO: review node-fetch options
             var request_options = prepareRequestOptions({
+                
+                timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+                jar: jar,
+                redirect: options.followRedirect ? 'follow' : 'manual',
+                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
+
+                // Reviewed.
                 uri: url,
                 method: 'GET',
                 headers: {
@@ -193,12 +200,6 @@ export function getUrlFunctional(url, options, callbacks) {
                     'Accept': '*/*'
                     // gzip, br, etc - set by `node-fetch` default `compress: true` option.
                 },
-                timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-                jar: jar,
-
-                // New for `node-fetch`
-                redirect: options.followRedirect ? 'follow' : 'manual',
-                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
                 createAbortController: true
             }, options);
 
@@ -260,22 +261,22 @@ var getHead = function(url, options) {
 
             // TODO: review node-fetch options
             var request_options = prepareRequestOptions({
-                uri: url,
-                method: 'HEAD',
-                headers: {
-                    'User-Agent': CONFIG.USER_AGENT,
-                    //'Connection': 'close'
-                },
                 timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
                 jar: jar,
                 agentOptions: {
                     // getHead called for video, need check certificate.
                     rejectUnauthorized: true
                 },
-
-                // New for `node-fetch`.
                 redirect: options.followRedirect ? 'follow' : 'manual',
-                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
+                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
+
+                // Reviewed.
+                uri: url,
+                method: 'HEAD',
+                headers: {
+                    'User-Agent': CONFIG.USER_AGENT,
+                    //'Connection': 'close'
+                },
                 // No abort controller for head.
             }, {
                 disableHttp2: options && options.disableHttp2
@@ -898,9 +899,11 @@ export function sendLogToWhitelist(uri, context) {
 
         // TODO: check options
         fetchStream({
+            qs: data,
+
+            // Reviewed.
             uri: CONFIG.WHITELIST_LOG_URL,
             method: 'GET',
-            qs: data
         })
             .then(res => {
                 if (res.status !== 200) {
@@ -1067,17 +1070,18 @@ export function generateLinksHtml(data, options) {
 var getUriStatusPrivate = function(uri, options, cb) {
 
     var request_options = prepareRequestOptions({
+        
+        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+        jar: request.jar(),   //Enable cookies, uses new jar
+        redirect: 'follow', // Default value.
+        follow: CONFIG.MAX_REDIRECTS,
+
+        // Reviewed.
         uri: uri,
         method: 'GET',
         headers: {
             'User-Agent': CONFIG.USER_AGENT
         },
-        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        jar: request.jar(),   //Enable cookies, uses new jar
-
-        // New by 'node-fetch'.
-        redirect: 'follow', // Default value.
-        follow: CONFIG.MAX_REDIRECTS,
         createAbortController: true
     }, {
         disableHttp2: options.disableHttp2

From c651a370966c75946355c2febc9f1c75aeecba2c Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 21 Oct 2021 21:27:35 +0300
Subject: [PATCH 043/102] bugfix typo `this.result` -> `this._result`

---
 lib/plugins/system/meta/HTMLMetaHandler.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/plugins/system/meta/HTMLMetaHandler.js b/lib/plugins/system/meta/HTMLMetaHandler.js
index 2f86811fe..abbb66886 100644
--- a/lib/plugins/system/meta/HTMLMetaHandler.js
+++ b/lib/plugins/system/meta/HTMLMetaHandler.js
@@ -227,7 +227,7 @@ HTMLMetaHandler.prototype.onend = function() {
 HTMLMetaHandler.prototype._finalMerge = function(tag) {
 
     if (tag === 'HEAD' 
-        && (!this.result || !this.result.og && !this.result.twitter && !this.result['ld-json'])) {
+        && (!this._result || !this._result.og && !this._result.twitter && !this._result['ld-json'])) {
         // No useful meta in HEAD, let's dig into the body.
         // Ex.: YouTube /c/ channels
         return;

From c95dc95eedd041a1d5df915e1187fe74c84c1c15 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 22 Oct 2021 21:47:05 +0300
Subject: [PATCH 044/102] api request: user default user agent

---
 lib/request.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/request.js b/lib/request.js
index dd2cca0b6..c639a7468 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -141,6 +141,11 @@ export default function(options, iframely_options, callback) {
             });
         }
 
+        if (!options.headers || !options.headers['User-Agent']) {
+            options.headers = options.headers || {};
+            options.headers['User-Agent'] = CONFIG.USER_AGENT;
+        }
+
         fetchData(utils.prepareRequestOptions(options, iframely_options))
             .then(result => {
                 finish(null, result);

From fd3ac1a02f239679258f60eee3a2b27002995d2e Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 22 Oct 2021 21:49:58 +0300
Subject: [PATCH 045/102] fetch: remove obsolete EventEmitter; use
 `keep-alive`; remove obsolete `secondAttempt`

---
 lib/core.js                                 |   4 +-
 lib/fetch.js                                |  18 +-
 lib/plugins/system/htmlparser/htmlparser.js |   6 +-
 lib/plugins/system/oembed/oembedUtils.js    |   4 +-
 lib/request.js                              |   2 -
 lib/utils.js                                | 261 +++++++-------------
 lib/whitelist.js                            |   1 +
 utils.js                                    |   2 +-
 8 files changed, 113 insertions(+), 185 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index 86fe10f76..17beb5744 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -1,6 +1,5 @@
 
     import * as _ from 'underscore';
-    import request from 'request';
     import * as urlLib from 'url';
     import * as pluginUtils from './loader/utils.js';
     import * as utils from './utils.js';
@@ -1426,7 +1425,8 @@
         };
 
         if (!options.jar) {
-            options.jar = request.jar();
+            // TODO: 
+            //options.jar = request.jar();
         }
 
         if ('disableCache' in options) {
diff --git a/lib/fetch.js b/lib/fetch.js
index b28145ce5..a68a47420 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,8 +1,14 @@
 import { noCache, timeoutSignal, AbortController, createUrl } from '@adobe/helix-fetch';
 
+const { fetchKeepAlive } = noCache({
+    h1: {
+        keepAlive: true
+    }
+});
+
 const { fetch } = noCache();
 
-export function fetchStream(options) {
+function doFetch(fetch_func, options) {
     const createAbortController = options.createAbortController;
     const abortController = createAbortController ? new AbortController() : null;
     const fetch_options = Object.assign({}, options);
@@ -11,7 +17,7 @@ export function fetchStream(options) {
     }
     const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
     return new Promise((resolve, reject) => {
-        fetch(uri, fetch_options)
+        fetch_func(uri, fetch_options)
             .then(response => {
                 var stream = response.body;
                 stream.status = response.status;
@@ -21,6 +27,14 @@ export function fetchStream(options) {
             })
             .catch(reject);
     });
+}
+
+export function fetchStreamKeepAlive(options) {
+    doFetch(fetchKeepAlive, options);
+}
+
+export function fetchStream(options) {
+    doFetch(fetch, options);
 };
 
 export function fetchData(options) {
diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 4a11dbf60..34e2a0870 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -34,10 +34,10 @@ export default {
                     });
                 }
             },
-            onAbortController: function(controller) {
-                abortController = controller;
-            },
             onResponse: function(resp) {
+
+                abortController = resp.abortController;
+
                 function cacheAndRespond(error, data, preventCache) {
 
                     var skip_cache = preventCache;
diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js
index 47e9c3bef..a59f9187b 100644
--- a/lib/plugins/system/oembed/oembedUtils.js
+++ b/lib/plugins/system/oembed/oembedUtils.js
@@ -227,9 +227,7 @@ export function findOembedLinks(uri, meta) {
                 });
             }
 
-            getUrlFunctional(uri, {
-                disableHttp2: options && options.disableHttp2
-            }, {
+            getUrlFunctional(uri, {}, {
                 onError: cbWrapper,
                 onResponse: function(res) {
                     if (res.status == 200) {
diff --git a/lib/request.js b/lib/request.js
index c639a7468..63e4cea60 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -68,8 +68,6 @@ export default function(options, iframely_options, callback) {
     delete options.new_cache_key;
     delete options.refresh;
 
-    options.gzip = true;
-
     var prefix = CONFIG.API_REQUEST_CACHE_PREFIX ? (CONFIG.API_REQUEST_CACHE_PREFIX + ':') : '';
 
     var lang = iframely_options.getProviderOptions('locale');
diff --git a/lib/utils.js b/lib/utils.js
index 1f0ea47a9..991773a8f 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,6 +1,5 @@
 import * as events from 'events';
-import request from 'request';
-import { fetchStream } from './fetch.js';
+import { fetchStream, fetchStreamKeepAlive } from './fetch.js';
 import * as iconv from 'iconv-lite';
 import * as async from 'async';
 import imagesize from 'probe-image-size';
@@ -75,8 +74,6 @@ export function prepareRequestOptions(request_options, options) {
     var uri = request_options.uri;
     delete request_options.url;
 
-    var disableHttp2 = options && options.disableHttp2 || CONFIG.DISABLE_HTTP2;
-
     if (CONFIG.PROXY || (options && options.proxy)) {
 
         var proxy = (options && options.proxy) || CONFIG.PROXY.find(p => {
@@ -101,24 +98,9 @@ export function prepareRequestOptions(request_options, options) {
             if (proxy.request_options) {
                 _.extend(request_options, proxy.request_options);
             }
-            if (proxy.disable_http2) {
-                disableHttp2 = true;
-            }
         }
     }
 
-    if (!disableHttp2
-        && !request_options.proxy
-        && /^https:/.test(uri)
-        && (!request_options.method
-            || request_options.method === 'GET'
-            || request_options.method === 'HEAD')) {
-        // http2 only for ssl and without proxy.
-        
-        // TODO:
-        // request_options.agent = http2Agent;
-    }
-
     var lang = options && options.getProviderOptions && options.getProviderOptions('locale', 'en-US');
     if (lang) {
         request_options.headers = request_options.headers || {};
@@ -130,34 +112,6 @@ export function prepareRequestOptions(request_options, options) {
     return request_options;
 };
 
-export function getUrlFunctional(url, options, callbacks) {
-
-    getUrl(url, options)
-        .on('error', function(error, opts) {
-
-            // Retry on ECONNRESET. Http2Agent will try with `http1` after this error.
-            if (!options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                log('   -- getUrl ' + error.code + ' first', opts, url);
-                process.nextTick(function() {
-                    getUrlFunctional(url, _.extend({}, options, {
-                        secondAttempt: true, 
-                        disableHttp2: true
-                    }), callbacks);
-                });
-                return;
-            } else if (options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                log('   -- getUrl ' + error.code + ' second', opts, url);
-            }
-
-            callbacks.onError && callbacks.onError(error);
-        })
-        .on('response', function(response) {
-            // TODO: remove abort event
-            callbacks.onAbortController && callbacks.onAbortController(response.abortController);
-            callbacks.onResponse && callbacks.onResponse(response);
-        });
-}
-
 /**
  * @public
  * Do HTTP GET request and handle redirects
@@ -168,136 +122,103 @@ export function getUrlFunctional(url, options, callbacks) {
  * @param {Function} [callback] The completion callback function or events.EventEmitter object
  * @returns {events.EventEmitter} The emitter object which emit error or response event
  */
- export function getUrl(url, options) {
-
-    var req = new events.EventEmitter();
+ export function getUrl(url, options, callbacks) {
 
     var options = options || {};
 
+    // TODO:
     // Store cookies between redirects and requests.
-    var jar = options.jar;
-    if (!jar) {
-        jar = request.jar();
-    }
-
-    process.nextTick(function() {
-        try {
+    // var jar = options.jar;
+    // if (!jar) {
+    //     jar = request.jar();
+    // }
 
-            // TODO: review node-fetch options
-            var request_options = prepareRequestOptions({
-                
-                timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-                jar: jar,
-                redirect: options.followRedirect ? 'follow' : 'manual',
-                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
-
-                // Reviewed.
-                uri: url,
-                method: 'GET',
-                headers: {
-                    'User-Agent': options.user_agent || CONFIG.USER_AGENT,
-                    //'Connection': 'keep-alive',                    
-                    'Accept': '*/*'
-                    // gzip, br, etc - set by `node-fetch` default `compress: true` option.
-                },
-                createAbortController: true
-            }, options);
+    // TODO: review node-fetch options
+    var request_options = prepareRequestOptions({
+        
+        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+        // jar: jar,
+        
+        // Reviewed.
+        uri: url,
+        method: 'GET',
+        headers: {
+            'User-Agent': options.user_agent || CONFIG.USER_AGENT,
+            'Accept': '*/*'
+        },
+        redirect: options.followRedirect ? 'follow' : 'manual',
+        follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
+        createAbortController: true
+    }, options);
 
-            fetchStream(request_options)
-                .then(stream => {
-                    if (!options.asBuffer) {
-                        stream.setEncoding("binary");
-                    }
-                    req.emit('response', stream);
-                })
-                .catch(error => {
-                    req.emit('error', error, {http2Agent: false/*request_options.agent === http2Agent*/});
-                });
+    try {
+        fetchStreamKeepAlive(request_options)
+            .then(stream => {
+                if (!options.asBuffer) {
+                    stream.setEncoding("binary");
+                }
+                callbacks.onResponse && callbacks.onResponse(stream);
+            })
+            .catch(error => {
+                callbacks.onError && callbacks.onError(error);
+            });
 
-        } catch (ex) {
-            console.error('Error on getUrl for', url, '.\n Error:' + ex);
-            req.emit('error', ex);
-        }
-    });
-    return req;
+    } catch (ex) {
+        console.error('Error on getUrl for', url, '.\n Error:' + ex);
+        callbacks.onError && callbacks.onError(ex);
+    }
 };
 
-export function getHeadFunctional(url, options, callbacks) {
-    getHead(url, options)
-        .on('error', function(error) {
+export const getUrlFunctional = getUrl;
 
-            // Retry on ECONNRESET.
-            if (!options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                log('   -- getHead ' + error.code + ' first', url);
-                process.nextTick(function() {
-                    getHeadFunctional(url, _.extend({}, options, {secondAttempt: true}), callbacks);
-                });
-                return;
-            } else if (options.secondAttempt && (error.code in CONFIG.HTTP2_RETRY_CODES)) {
-                log('   -- getHead ' + error.code + ' second', url);
-            }
-
-            callbacks.onError && callbacks.onError(error);
-        })
-        .on('response', function(response) {
-            callbacks.onResponse && callbacks.onResponse(response);
-        });
-}
-
-var getHead = function(url, options) {
-
-    var req = new events.EventEmitter();
+var getHead = function(url, options, callbacks) {
 
     var options = options || {};
 
+    // TODO: 
     // Store cookies between redirects and requests.
-    var jar = options.jar;
-    if (!jar) {
-        jar = request.jar();
-    }
+    // var jar = options.jar;
+    // if (!jar) {
+    //     jar = request.jar();
+    // }
 
-    process.nextTick(function() {
-        try {
+    // TODO: review node-fetch options
+    var request_options = prepareRequestOptions({
+        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+        // jar: jar,
+        agentOptions: {
+            // getHead called for video, need check certificate.
+            rejectUnauthorized: true
+        },
+        
+        // Reviewed.
+        uri: url,
+        method: 'HEAD',
+        headers: {
+            'User-Agent': CONFIG.USER_AGENT
+        },
+        redirect: options.followRedirect ? 'follow' : 'manual',
+        follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
+        // No abort controller for head.
+    });
 
-            // TODO: review node-fetch options
-            var request_options = prepareRequestOptions({
-                timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-                jar: jar,
-                agentOptions: {
-                    // getHead called for video, need check certificate.
-                    rejectUnauthorized: true
-                },
-                redirect: options.followRedirect ? 'follow' : 'manual',
-                follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
-
-                // Reviewed.
-                uri: url,
-                method: 'HEAD',
-                headers: {
-                    'User-Agent': CONFIG.USER_AGENT,
-                    //'Connection': 'close'
-                },
-                // No abort controller for head.
-            }, {
-                disableHttp2: options && options.disableHttp2
+    try {
+        fetchStream(request_options)
+            .then(response => {
+                callbacks.onResponse && callbacks.onResponse(response);
+            })
+            .catch(error => {
+                callbacks.onError && callbacks.onError(error);
             });
 
-            fetchStream(request_options)
-                .then(response => {
-                    req.emit('response', response);
-                })
-                .catch(error => {
-                    req.emit('error', error);
-                });
-
-        } catch (ex) {
-            console.error('Error on getHead for', url, '.\n Error:' + ex);
-            req.emit('error', ex);
-        }
-    });
-    return req;
+    } catch (ex) {
+        console.error('Error on getHead for', url, '.\n Error:' + ex);
+        callbacks.onError && callbacks.onError(ex);
+    }
 };
 
+export const getHeadFunctional = getHead;
+
 export function getCharset(string, doNotParse) {
     var charset;
 
@@ -515,14 +436,12 @@ export function parseJSONSource(text, decode) {
 
                 getUrlFunctional(uri, {
                     timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-                    asBuffer: true,
-                    disableHttp2: options && options.disableHttp2
+                    asBuffer: true
                 }, {
-                    onAbortController: function(controller) {
-                        abortController = controller;
-                    },
                     onResponse: function(res) {
 
+                        abortController = res.abortController;
+
                         var content_type = res.headers['content-type'];
 
                         if (content_type && content_type !== 'application/octet-stream' && content_type !== 'binary/octet-stream') {
@@ -698,13 +617,12 @@ export function getContentType(uriForCache, uriOriginal, options, cb) {
             var methodCaller = method && method === 'GET' ? getUrlFunctional : getHeadFunctional;
 
             methodCaller(uriOriginal, {
-                timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-                disableHttp2: options && options.disableHttp2
+                timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT
             }, {
-                onAbortController: function(controller) {
-                    abortController = controller;
-                },
                 onResponse: function(res) {
+
+                    abortController = res.abortController;
+
                     var error = res.status && res.status != 200 ? res.status : null;
 
                     if (!method
@@ -1072,9 +990,9 @@ var getUriStatusPrivate = function(uri, options, cb) {
     var request_options = prepareRequestOptions({
         
         timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        jar: request.jar(),   //Enable cookies, uses new jar
-        redirect: 'follow', // Default value.
-        follow: CONFIG.MAX_REDIRECTS,
+        // TODO:
+        //jar: request.jar(),   //Enable cookies, uses new jar
+        
 
         // Reviewed.
         uri: uri,
@@ -1082,9 +1000,8 @@ var getUriStatusPrivate = function(uri, options, cb) {
         headers: {
             'User-Agent': CONFIG.USER_AGENT
         },
+        follow: CONFIG.MAX_REDIRECTS,
         createAbortController: true
-    }, {
-        disableHttp2: options.disableHttp2
     })
 
     try {
diff --git a/lib/whitelist.js b/lib/whitelist.js
index eb738b239..20774a649 100644
--- a/lib/whitelist.js
+++ b/lib/whitelist.js
@@ -350,6 +350,7 @@
                     // Prevent caching.
                     'Cache-Control': 'no-cache'
                 },
+                // TODO: remove in helix-fetch
                 gzip: true
             };
 
diff --git a/utils.js b/utils.js
index 45b61cf93..50b9d6160 100644
--- a/utils.js
+++ b/utils.js
@@ -10,7 +10,7 @@
     import * as urlLib from 'url';
 
     import log from './logging.js';
-    export {log};
+    export { log };
     import * as whitelist from './lib/whitelist.js';
     import * as pluginLoader from './lib/loader/pluginLoader.js';
 

From 089bca6e8ae027902a085aafe2b83553e7b3b2fd Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 22 Oct 2021 21:55:34 +0300
Subject: [PATCH 046/102] bugfix fetch wrapper

---
 lib/fetch.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index a68a47420..085ff5119 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,10 +1,10 @@
 import { noCache, timeoutSignal, AbortController, createUrl } from '@adobe/helix-fetch';
 
-const { fetchKeepAlive } = noCache({
+const fetchKeepAlive = noCache({
     h1: {
         keepAlive: true
     }
-});
+}).fetch;
 
 const { fetch } = noCache();
 
@@ -30,11 +30,11 @@ function doFetch(fetch_func, options) {
 }
 
 export function fetchStreamKeepAlive(options) {
-    doFetch(fetchKeepAlive, options);
+    return doFetch(fetchKeepAlive, options);
 }
 
 export function fetchStream(options) {
-    doFetch(fetch, options);
+    return doFetch(fetch, options);
 };
 
 export function fetchData(options) {

From 421873792fe3ca9974cdfcd24a5eccb0a25752b4 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 22 Oct 2021 21:57:27 +0300
Subject: [PATCH 047/102] remove `DISABLE_HTTP2` from config sample

---
 config.local.js.SAMPLE | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/config.local.js.SAMPLE b/config.local.js.SAMPLE
index 61dac1ba2..b7d0c503a 100644
--- a/config.local.js.SAMPLE
+++ b/config.local.js.SAMPLE
@@ -121,13 +121,6 @@
         // If there's no response from remote server, the timeout will occur after
         RESPONSE_TIMEOUT: 5 * 1000, //ms
 
-        /* From v1.4.0, Iframely works over HTTP/2 if not disabled.
-           Alternatively, you can also disable per origin. See `proxy` option below.
-           The default in this sample config is to disable HTTP/2 as per
-           https://github.com/itteco/iframely/issues/277.
-        */
-        DISABLE_HTTP2: true,
-
         // Customize API calls to oembed endpoints.
         // Must have: please add your `access_token` for Facebook and Instagram API calls
         ADD_OEMBED_PARAMS: [{
@@ -196,8 +189,7 @@
                 // Refer to: https://github.com/request/request
                 // Overrides previous params if overlapped.
             },
-            cache_ttl: 3600, // in seconds, cache response for 1 hour.
-            disable_http2: true
+            cache_ttl: 3600  // in seconds, cache response for 1 hour.
         }],
         */
 

From ca72a7ed5d07d6cef6cbd94ce8e7e3e400308baf Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 25 Oct 2021 23:52:34 +0300
Subject: [PATCH 048/102] plugins: remove obsolete `request` options

---
 plugins/domains/sverigesradio.se.js      | 1 -
 plugins/domains/tumblr.com/tumblr.api.js | 1 -
 plugins/links/hosted/ooyala.js           | 1 -
 3 files changed, 3 deletions(-)

diff --git a/plugins/domains/sverigesradio.se.js b/plugins/domains/sverigesradio.se.js
index b162b678a..63fc17a2e 100644
--- a/plugins/domains/sverigesradio.se.js
+++ b/plugins/domains/sverigesradio.se.js
@@ -14,7 +14,6 @@ export default {
 
         request({
             uri: "http://sverigesradio.se/sida/playerajax/getaudiourl?id=" + urlMatch[1] + "&type=publication&quality=medium&format=iis",
-            limit: 1, 
             timeout: 1000,
             prepareResult: function(error, response, body, cb) {
 
diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js
index fce4b181a..f696f985a 100644
--- a/plugins/domains/tumblr.com/tumblr.api.js
+++ b/plugins/domains/tumblr.com/tumblr.api.js
@@ -75,7 +75,6 @@ export default {
                 id: urlMatch[3]
             },
             json: true,
-            limit: 1, 
             timeout: 1000,
             prepareResult: function (error, response, body, cb) {
 
diff --git a/plugins/links/hosted/ooyala.js b/plugins/links/hosted/ooyala.js
index 852fe93c6..5626e229d 100644
--- a/plugins/links/hosted/ooyala.js
+++ b/plugins/links/hosted/ooyala.js
@@ -38,7 +38,6 @@ export default {
             method: 'HEAD',
             headers: {
                 'Accept-Encoding': 'gzip, deflate, sdch', // this is required for head request to return content_length
-                'Connection': 'close'
             },
             prepareResult: function(error, response, body, cb) {
 

From 578ea719e3c41f655d211e6128ae7afbec80c78e Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 00:06:42 +0300
Subject: [PATCH 049/102] fetch: add timeout

---
 lib/fetch.js   | 26 +++++++++++++++++++-------
 lib/request.js |  1 -
 lib/utils.js   | 24 ++++++------------------
 3 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 085ff5119..ce5710edd 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,4 +1,4 @@
-import { noCache, timeoutSignal, AbortController, createUrl } from '@adobe/helix-fetch';
+import { noCache, createUrl, AbortController, AbortError } from '@adobe/helix-fetch';
 
 const fetchKeepAlive = noCache({
     h1: {
@@ -9,12 +9,15 @@ const fetchKeepAlive = noCache({
 const { fetch } = noCache();
 
 function doFetch(fetch_func, options) {
-    const createAbortController = options.createAbortController;
-    const abortController = createAbortController ? new AbortController() : null;
+    const abortController = new AbortController();
     const fetch_options = Object.assign({}, options);
-    if (abortController) {
-        fetch_options.signal = abortController.signal;
-    }
+    // Allow request abort before finish.
+    fetch_options.signal = abortController.signal;
+    // Implement `timeout`.
+    const timeoutTimerId = setTimeout(() => {
+        abortController.abort()
+    }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
+    // Implement `qs` (get params).
     const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
     return new Promise((resolve, reject) => {
         fetch_func(uri, fetch_options)
@@ -25,7 +28,16 @@ function doFetch(fetch_func, options) {
                 stream.abortController = abortController;
                 resolve(stream);
             })
-            .catch(reject);
+            .catch(error => {
+                if (error instanceof AbortError) {
+                    // `AbortError` before `response` occurs only on timeout.
+                    error = 'timeout';
+                }
+                reject(error);
+            })
+            .finally(() => {
+                clearTimeout(timeoutTimerId);
+            });
     });
 }
 
diff --git a/lib/request.js b/lib/request.js
index 63e4cea60..657c87bdd 100644
--- a/lib/request.js
+++ b/lib/request.js
@@ -36,7 +36,6 @@ options:
 
 jar: false,
 limit: 1, 
-timeout: 1000,
 
 oauth
 
diff --git a/lib/utils.js b/lib/utils.js
index 991773a8f..094ce338c 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -126,18 +126,8 @@ export function prepareRequestOptions(request_options, options) {
 
     var options = options || {};
 
-    // TODO:
-    // Store cookies between redirects and requests.
-    // var jar = options.jar;
-    // if (!jar) {
-    //     jar = request.jar();
-    // }
-
-    // TODO: review node-fetch options
     var request_options = prepareRequestOptions({
-        
-        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        // jar: jar,
+        // TODO: jar: jar,
         
         // Reviewed.
         uri: url,
@@ -146,9 +136,9 @@ export function prepareRequestOptions(request_options, options) {
             'User-Agent': options.user_agent || CONFIG.USER_AGENT,
             'Accept': '*/*'
         },
+        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
         redirect: options.followRedirect ? 'follow' : 'manual',
-        follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0,
-        createAbortController: true
+        follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
     }, options);
 
     try {
@@ -184,7 +174,6 @@ var getHead = function(url, options, callbacks) {
 
     // TODO: review node-fetch options
     var request_options = prepareRequestOptions({
-        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
         // jar: jar,
         agentOptions: {
             // getHead called for video, need check certificate.
@@ -197,6 +186,7 @@ var getHead = function(url, options, callbacks) {
         headers: {
             'User-Agent': CONFIG.USER_AGENT
         },
+        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
         redirect: options.followRedirect ? 'follow' : 'manual',
         follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
         // No abort controller for head.
@@ -988,8 +978,6 @@ export function generateLinksHtml(data, options) {
 var getUriStatusPrivate = function(uri, options, cb) {
 
     var request_options = prepareRequestOptions({
-        
-        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
         // TODO:
         //jar: request.jar(),   //Enable cookies, uses new jar
         
@@ -1000,8 +988,8 @@ var getUriStatusPrivate = function(uri, options, cb) {
         headers: {
             'User-Agent': CONFIG.USER_AGENT
         },
-        follow: CONFIG.MAX_REDIRECTS,
-        createAbortController: true
+        timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
+        follow: CONFIG.MAX_REDIRECTS
     })
 
     try {

From 7bb0b132f254a7a38d33b5b55669e4dc57390c02 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 00:16:10 +0300
Subject: [PATCH 050/102] utils: `rejectUnauthorized: true` for `getHead`
 request

---
 app.js       |  1 +
 lib/fetch.js | 13 +++++++++++--
 lib/utils.js | 10 +++-------
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/app.js b/app.js
index 94b4f5ad1..63c58c129 100644
--- a/app.js
+++ b/app.js
@@ -173,4 +173,5 @@ app.get('/', function(req, res) {
 });
 
 process.title = "iframely";
+// Not used by `helix-fetch`.
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
\ No newline at end of file
diff --git a/lib/fetch.js b/lib/fetch.js
index ce5710edd..d357da278 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -3,10 +3,15 @@ import { noCache, createUrl, AbortController, AbortError } from '@adobe/helix-fe
 const fetchKeepAlive = noCache({
     h1: {
         keepAlive: true
-    }
+    },
+    rejectUnauthorized: false   // By default skip auth check for all.
 }).fetch;
 
-const { fetch } = noCache();
+const fetchAuthorized = noCache().fetch;    // `rejectUnauthorized: true` - by `fetch` default.
+
+const { fetch } = noCache({
+    rejectUnauthorized: false   // By default skip auth check for all.
+});
 
 function doFetch(fetch_func, options) {
     const abortController = new AbortController();
@@ -49,6 +54,10 @@ export function fetchStream(options) {
     return doFetch(fetch, options);
 };
 
+export function fetchStreamAuthorized(options) {
+    return doFetch(fetchAuthorized, options);
+};
+
 export function fetchData(options) {
     var json = options.json;
     var response;
diff --git a/lib/utils.js b/lib/utils.js
index 094ce338c..8f613893e 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,5 +1,5 @@
 import * as events from 'events';
-import { fetchStream, fetchStreamKeepAlive } from './fetch.js';
+import { fetchStream, fetchStreamKeepAlive, fetchStreamAuthorized } from './fetch.js';
 import * as iconv from 'iconv-lite';
 import * as async from 'async';
 import imagesize from 'probe-image-size';
@@ -175,11 +175,7 @@ var getHead = function(url, options, callbacks) {
     // TODO: review node-fetch options
     var request_options = prepareRequestOptions({
         // jar: jar,
-        agentOptions: {
-            // getHead called for video, need check certificate.
-            rejectUnauthorized: true
-        },
-        
+
         // Reviewed.
         uri: url,
         method: 'HEAD',
@@ -193,7 +189,7 @@ var getHead = function(url, options, callbacks) {
     });
 
     try {
-        fetchStream(request_options)
+        fetchStreamAuthorized(request_options)
             .then(response => {
                 callbacks.onResponse && callbacks.onResponse(response);
             })

From f2c95b097d34f88f7e098c235d1e6e6df199d849 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 16:06:30 +0300
Subject: [PATCH 051/102] core: return used http2 protocol

---
 lib/core.js                                 | 5 +----
 lib/fetch.js                                | 1 +
 lib/plugins/system/htmlparser/htmlparser.js | 8 +++-----
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index 17beb5744..dc1b8ae41 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -880,11 +880,8 @@
                         } else {
 
                             if (key === 'htmlparser') {
-
                                 // Debug http2 state.
-    
-                                // TODO: detect h2
-                                //allResults.h2 = r.data.htmlparser.request.response.h2;
+                                allResults.h2 = r.data.htmlparser.h2;
                             }
 
                             // First data has priority, do not override it.
diff --git a/lib/fetch.js b/lib/fetch.js
index d357da278..4552ed742 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -31,6 +31,7 @@ function doFetch(fetch_func, options) {
                 stream.status = response.status;
                 stream.headers = response.headers.plain();
                 stream.abortController = abortController;
+                stream.h2 = response.httpVersion === '2.0';
                 resolve(stream);
             })
             .catch(error => {
diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 34e2a0870..0dd15e4dd 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -18,8 +18,6 @@ export default {
 
     getData: function(url, whitelistRecord, options, __noCachedHtmlparserFallback, cb) {
 
-        var abortController;
-
         getUrlFunctional(url, _.extend({}, options, {
             followRedirect: !!options.followHTTPRedirect
         }), {
@@ -36,8 +34,6 @@ export default {
             },
             onResponse: function(resp) {
 
-                abortController = resp.abortController;
-
                 function cacheAndRespond(error, data, preventCache) {
 
                     var skip_cache = preventCache;
@@ -85,7 +81,8 @@ export default {
                     }
                 }
 
-                var headers = resp.headers;
+                const headers = resp.headers;
+                const abortController = resp.abortController;
 
                 if (resp.status >= 300 && resp.status< 400 && headers.location) {
                     abortController.abort();
@@ -145,6 +142,7 @@ export default {
                 });
                 handler.headers = headers;
                 handler.abortController = abortController;
+                handler.h2 = resp.h2;
 
                 // Do before resume?
                 cb(null, {

From 430407a848d9856716f085eaa43e3a61b488f212 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 16:20:34 +0300
Subject: [PATCH 052/102] tests: fix imports

---
 lib/loader/pluginLoader.js |  9 +++++----
 modules/tests-ui/models.js |  2 ++
 modules/tests-ui/tester.js | 10 ++++------
 modules/tests-ui/utils.js  |  2 +-
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js
index edbc3274d..2a3ffc86f 100644
--- a/lib/loader/pluginLoader.js
+++ b/lib/loader/pluginLoader.js
@@ -341,11 +341,12 @@
             for(var i = 0; i < mixins.length; i++) {
 
                 var mixin = mixins[i];
+                if (plugins[mixin]) {
+                    var m = plugins[mixin].getPluginLastModifiedDate();
 
-                var m = plugins[mixin].getPluginLastModifiedDate();
-
-                if (m > modified) {
-                    modified = m;
+                    if (m > modified) {
+                        modified = m;
+                    }
                 }
             }
         }
diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js
index b11bcbfc4..87e9a31fc 100644
--- a/modules/tests-ui/models.js
+++ b/modules/tests-ui/models.js
@@ -1,4 +1,6 @@
     import * as moment from 'moment';
+    import config from '../../config.js';
+    global.CONFIG = config;
 
     var mongoose, db;
 
diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js
index dbcd8b7e7..3d0b6c913 100644
--- a/modules/tests-ui/tester.js
+++ b/modules/tests-ui/tester.js
@@ -1,12 +1,10 @@
-import config from '../../config';
+import config from '../../config.js';
 global.CONFIG = config;
 
-global.CONFIG = config.default;
-
 if (!CONFIG.tests) {
     console.error('Tests not started: CONFIG.tests not configured.');
     process.exit(0);
-    return;
+    // return;
 }
 
 process.title = "iframely-tests";
@@ -15,7 +13,7 @@ import * as async from 'async';
 import * as _ from 'underscore';
 import * as models from './models.js';
 import * as utils from './utils.js';
-import { run as iframely } from '../../lib/core';
+import { run as iframely } from '../../lib/core.js';
 import * as whitelist from '../../lib/whitelist.js';
 import * as pluginLoader from '../../lib/loader/pluginLoader.js';
 var plugins = pluginLoader._plugins;
@@ -51,7 +49,7 @@ var TestingProgress = models.TestingProgress;
 
 if (!PluginTest) {
     process.exit(0);
-    return;
+    // return;
 }
 
 function log() {
diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
index b4684ddaf..a47915a58 100644
--- a/modules/tests-ui/utils.js
+++ b/modules/tests-ui/utils.js
@@ -1,5 +1,5 @@
 import * as _ from 'underscore';
-import * as FeedParser from 'feedparser';
+import FeedParser from 'feedparser';
 import request from 'request';
 import * as async from 'async';
 import * as url from 'url';

From f377db8323510dbfd5833a7a06b375b7d578f45b Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 19:13:18 +0300
Subject: [PATCH 053/102] tests: update imports

---
 modules/tests-ui/models.js | 23 +++++++----------------
 modules/tests-ui/tester.js |  5 -----
 modules/tests-ui/utils.js  |  7 ++++++-
 3 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js
index 87e9a31fc..598a9e1f1 100644
--- a/modules/tests-ui/models.js
+++ b/modules/tests-ui/models.js
@@ -1,24 +1,15 @@
     import * as moment from 'moment';
+    import mongoose from 'mongoose';
     import config from '../../config.js';
     global.CONFIG = config;
 
-    var mongoose, db;
-
-    // DB connect.
-    try {
-        const mongoosePkg = await import('mongoose');
-        mongoose = mongoosePkg.default;
-        mongoose.set('useUnifiedTopology', true);
-        mongoose.set('useCreateIndex', true);
-        mongoose.set('useNewUrlParser', true);
-        if (global.Promise) {
-            mongoose.Promise = global.Promise;
-        }
-        db = mongoose.createConnection(CONFIG.tests.mongodb);
-    } catch (ex) {
-        console.error("Plugins testing framework will not work. Can't connect to mongodb.");
-        console.error(ex.stack);
+    mongoose.set('useUnifiedTopology', true);
+    mongoose.set('useCreateIndex', true);
+    mongoose.set('useNewUrlParser', true);
+    if (global.Promise) {
+        mongoose.Promise = global.Promise;
     }
+    const db = mongoose.createConnection(CONFIG.tests.mongodb);
 
     var Schema = mongoose.Schema;
 
diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js
index 3d0b6c913..44b7b5623 100644
--- a/modules/tests-ui/tester.js
+++ b/modules/tests-ui/tester.js
@@ -47,11 +47,6 @@ var PageTestLog = models.PageTestLog;
 var TestUrlsSet = models.TestUrlsSet;
 var TestingProgress = models.TestingProgress;
 
-if (!PluginTest) {
-    process.exit(0);
-    // return;
-}
-
 function log() {
     if (CONFIG.DEBUG) {
         console.log.apply(console, arguments);
diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
index a47915a58..de29ce6ab 100644
--- a/modules/tests-ui/utils.js
+++ b/modules/tests-ui/utils.js
@@ -260,7 +260,12 @@ export function fetchFeedUrls(feedUrl, options, cb) {
         cb(error, urls);
     };
 
-    request(feedUrl)
+    request({
+        uri: feedUrl,
+        agentOptions: {
+            rejectUnauthorized: false
+        }
+    })
         .pipe(new FeedParser({addmeta: false}))
         .on('error', function(error) {
             _cb(error);

From 004b1649aa606ad9199c6a071b15c741995e6a46 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 19:13:39 +0300
Subject: [PATCH 054/102] config: update sample ssl usage with async file load

---
 config.local.js.SAMPLE | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/config.local.js.SAMPLE b/config.local.js.SAMPLE
index b7d0c503a..d48e3e5e6 100644
--- a/config.local.js.SAMPLE
+++ b/config.local.js.SAMPLE
@@ -1,9 +1,16 @@
     import { fileURLToPath } from 'url';
     import { dirname } from 'path';
+    import { readFile } from 'fs/promises';
 
     const __filename = fileURLToPath(import.meta.url);
     const __dirname = dirname(__filename);
 
+    // Optional SSL cert, if you serve under HTTPS.
+    /*
+    const ssl_key = const file = await readFile(new URL('./key.pem', import.meta.url), {encoding: 'utf8'});
+    const ssl_cert = await readFile(new URL('./cert.pem', import.meta.url), {encoding: 'utf8'});
+    */
+
     export default {
 
         // Specify a path for custom plugins. Custom plugins will override core plugins.
@@ -49,8 +56,8 @@
         // Optional SSL cert, if you serve under HTTPS.
         /*
         ssl: {
-            key: require('fs').readFileSync(__dirname + '/key.pem'),
-            cert: require('fs').readFileSync(__dirname + '/cert.pem'),
+            key: ssl_key,
+            cert: ssl_cert,
             port: 443
         },
         */

From 597bc046a1d4d54cb633a541b1001ca06dcc9c35 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 19:13:58 +0300
Subject: [PATCH 055/102] mamcached: fix import

---
 lib/cache-engines/memcached.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/cache-engines/memcached.js b/lib/cache-engines/memcached.js
index 6a30f244d..1e4c0d862 100644
--- a/lib/cache-engines/memcached.js
+++ b/lib/cache-engines/memcached.js
@@ -1,6 +1,7 @@
 import log from '../../logging.js';
 import * as crypto from 'crypto';
-import * as Memcached from 'memcached';
+import Memcached from 'memcached';
+import CONFIG from '../../config.js';
 
 var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options);
 

From 0dd47b204362bb41cf6599e85327c4d327e36f34 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 19:14:10 +0300
Subject: [PATCH 056/102] server: fix import https

---
 server.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server.js b/server.js
index a9c8152dd..48d52b107 100644
--- a/server.js
+++ b/server.js
@@ -1,5 +1,6 @@
 import * as sysUtils from './utils.js';
 import app from './app.js';
+import *  as https from 'https';
 
 var server = app.listen(process.env.PORT || CONFIG.port, process.env.HOST || CONFIG.host, function(){
     console.log('\niframely is running on ' + server.address().address + ':' + server.address().port);
@@ -7,7 +8,7 @@ var server = app.listen(process.env.PORT || CONFIG.port, process.env.HOST || CON
 });
 
 if (CONFIG.ssl) {
-    require('https').createServer(CONFIG.ssl, app).listen(CONFIG.ssl.port);
+    https.createServer(CONFIG.ssl, app).listen(CONFIG.ssl.port);
 }
 
 console.log('');

From 6db1fb516b9e584f168a15553acf01bbeede13dc Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 19:14:24 +0300
Subject: [PATCH 057/102] remove global
 process.env.NODE_TLS_REJECT_UNAUTHORIZED

---
 app.js                     | 4 +---
 modules/tests-ui/tester.js | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/app.js b/app.js
index 63c58c129..75d0c509a 100644
--- a/app.js
+++ b/app.js
@@ -172,6 +172,4 @@ app.get('/', function(req, res) {
   res.end();
 });
 
-process.title = "iframely";
-// Not used by `helix-fetch`.
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
\ No newline at end of file
+process.title = "iframely";
\ No newline at end of file
diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js
index 44b7b5623..6fd6a1546 100644
--- a/modules/tests-ui/tester.js
+++ b/modules/tests-ui/tester.js
@@ -8,7 +8,7 @@ if (!CONFIG.tests) {
 }
 
 process.title = "iframely-tests";
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+
 import * as async from 'async';
 import * as _ from 'underscore';
 import * as models from './models.js';

From 0016d60f3b6675945c3f42fcc117f5ab4f7861ad Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 19:20:46 +0300
Subject: [PATCH 058/102] fix redis import

---
 lib/cache-engines/redis.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/cache-engines/redis.js b/lib/cache-engines/redis.js
index a4932e8e2..dd91214d9 100644
--- a/lib/cache-engines/redis.js
+++ b/lib/cache-engines/redis.js
@@ -1,13 +1,15 @@
     import log from '../../logging.js';
+    import CONFIG from '../../config.js';
 
     var client;
 
     if (CONFIG.REDIS_MODE === 'cluster') {
-        var pkg = await import('redis-clustr');
-        client = new pkg.RedisClustr(CONFIG.REDIS_CLUSTER_OPTIONS);
+        const pkg = await import('redis-clustr');
+        const RedisClustr = pkg.default;
+        client = new RedisClustr(CONFIG.REDIS_CLUSTER_OPTIONS);
     } else {
         var pkg = await import('redis');
-        client = pkg.redis.createClient(CONFIG.REDIS_OPTIONS);
+        client = pkg.createClient(CONFIG.REDIS_OPTIONS);
     }
 
     export function set(key, data, options) {

From f1e08f7512cbacff773443726cd8088e04ab5c99 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 20:55:38 +0300
Subject: [PATCH 059/102] fix imports

---
 modules/tests-ui/models.js                     | 3 ++-
 modules/tests-ui/views.js                      | 2 +-
 plugins/domains/instagram.com/instagram.com.js | 2 +-
 plugins/domains/xkcd.com.js                    | 2 +-
 plugins/links/embedURL/embedURL.js             | 2 +-
 5 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js
index 598a9e1f1..f54a08f23 100644
--- a/modules/tests-ui/models.js
+++ b/modules/tests-ui/models.js
@@ -1,4 +1,4 @@
-    import * as moment from 'moment';
+    import moment from 'moment';
     import mongoose from 'mongoose';
     import config from '../../config.js';
     global.CONFIG = config;
@@ -9,6 +9,7 @@
     if (global.Promise) {
         mongoose.Promise = global.Promise;
     }
+
     const db = mongoose.createConnection(CONFIG.tests.mongodb);
 
     var Schema = mongoose.Schema;
diff --git a/modules/tests-ui/views.js b/modules/tests-ui/views.js
index d7785228a..ba411589e 100644
--- a/modules/tests-ui/views.js
+++ b/modules/tests-ui/views.js
@@ -1,5 +1,5 @@
     import * as async from 'async';
-    import * as moment from 'moment';
+    import moment from 'moment';
     import * as _ from 'underscore';
     import { exec as exec } from 'child_process';
     import * as models from './models.js';
diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js
index 9e4ef51ab..7276ac95e 100644
--- a/plugins/domains/instagram.com/instagram.com.js
+++ b/plugins/domains/instagram.com/instagram.com.js
@@ -1,6 +1,6 @@
 import cheerio_pkg from 'cheerio';
 const cheerio = cheerio_pkg.default;
-import { decodeHTML5 as decodeHTML5 } from 'entities';
+import { decodeHTML5 } from 'entities';
 
 export default {
 
diff --git a/plugins/domains/xkcd.com.js b/plugins/domains/xkcd.com.js
index be1f2cbc3..76ea42fe0 100644
--- a/plugins/domains/xkcd.com.js
+++ b/plugins/domains/xkcd.com.js
@@ -1,4 +1,4 @@
-import { decodeHTML5 as decodeHTML5 } from 'entities';
+import { decodeHTML5 } from 'entities';
 
 export default {
 
diff --git a/plugins/links/embedURL/embedURL.js b/plugins/links/embedURL/embedURL.js
index 6020bbe90..68ef34bf2 100644
--- a/plugins/links/embedURL/embedURL.js
+++ b/plugins/links/embedURL/embedURL.js
@@ -1,4 +1,4 @@
-import { decodeHTML5 as decodeHTML5 } from 'entities';
+import { decodeHTML5 } from 'entities';
 import * as utils from '../../../lib/utils.js';
 
 export default {

From 5871e8d83789fd23152801676b59f72e66b641f3 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 22:09:24 +0300
Subject: [PATCH 060/102] slideshare: fix merge

---
 plugins/domains/slideshare.net.js | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js
index 7cb6dc463..54a05c50d 100644
--- a/plugins/domains/slideshare.net.js
+++ b/plugins/domains/slideshare.net.js
@@ -1,4 +1,6 @@
 import * as utils from '../../lib/utils.js';
+import cheerio_pkg from 'cheerio';
+const $ = cheerio_pkg.default;
 
 export default {
 
@@ -29,9 +31,10 @@ export default {
 
     },
 
-    getLink: function(oembed, iframe, options, cb) {
+    getLink: function(oembed, options, cb) {
 
-        if (iframe.src && oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) {
+        if (oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) {
+            var links = [];
 
             var firstSlide = (/^\/\//.test(oembed.slide_image_baseurl) ? 'http:' : '') + oembed.slide_image_baseurl + '1' + oembed.slide_image_baseurl_suffix;
 
@@ -71,6 +74,16 @@ export default {
                     });
                 }
 
+                links.push ({
+                    href: oembed.thumbnail,
+                    type: CONFIG.T.image,
+                    rel: [CONFIG.R.thumbnail, CONFIG.R.oembed],
+                    width: oembed.thumbnail_width,
+                    height: data.height ? Math.round (oembed.thumbnail_width / (data.width / data.height)) : oembed.thumbnail_height
+                });
+
+                cb(null, links);
+
             });
         } else {
             cb(null, null);

From 137c501144d9baa5777a57be0188cb5e3983f95a Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 26 Oct 2021 22:11:47 +0300
Subject: [PATCH 061/102] slideshare: revert wrong merge

---
 plugins/domains/slideshare.net.js | 77 ++++++++++++-------------------
 1 file changed, 30 insertions(+), 47 deletions(-)

diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js
index 54a05c50d..42a686ecd 100644
--- a/plugins/domains/slideshare.net.js
+++ b/plugins/domains/slideshare.net.js
@@ -1,6 +1,4 @@
 import * as utils from '../../lib/utils.js';
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
 
 export default {
 
@@ -31,59 +29,44 @@ export default {
 
     },
 
-    getLink: function(oembed, options, cb) {
+    getLink: function(oembed, iframe, options, cb) {
 
-        if (oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) {
-            var links = [];
+        if (iframe.src && oembed.slide_image_baseurl && oembed.slide_image_baseurl_suffix) {
 
             var firstSlide = (/^\/\//.test(oembed.slide_image_baseurl) ? 'http:' : '') + oembed.slide_image_baseurl + '1' + oembed.slide_image_baseurl_suffix;
 
             utils.getImageMetadata(firstSlide, options, function(error, data) {
 
-                if (error || data.error) {
-
-                    console.log ('Error getting first slide for Slideshare: ' + error);
-
-                } else if (data.width && data.height) {
-
-                    links.push({
-                        href: firstSlide,
-                        type: CONFIG.T.image, 
-                        rel: CONFIG.R.thumbnail,
-                        width: data.width,
-                        height: data.height
-                    });
-                }
-
-                var $container = $('<div>');
-                try {
-                    $container.html(oembed.html);
-                } catch(ex) {}
-
-                var $iframe = $container.find('iframe');
-
-                var aspect = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height;
-
-                if ($iframe.length == 1) {
-                    links.push({
-                        href: $iframe.attr('src').replace('http:', ''),
-                        type: CONFIG.T.text_html,
-                        rel: [aspect > 1 ? CONFIG.R.player : CONFIG.R.reader, CONFIG.R.slideshow, CONFIG.R.html5],
-                        "aspect-ratio": aspect,
-                        "padding-bottom": 58
-                    });
+                if (error || data.error || !data.width || !data.height) {
+
+                    return cb('Error getting first slide for Slideshare: ' + error);
+
+                } else {
+
+                    var aspect = (data.width && data.height) ? data.width / data.height : oembed.width / oembed.height;
+
+                    return cb(null, [{
+                            href: firstSlide,
+                            type: CONFIG.T.image, 
+                            rel: CONFIG.R.thumbnail,
+                            width: data.width,
+                            height: data.height
+                        }, {
+                            href: oembed.thumbnail,
+                            type: CONFIG.T.image,
+                            rel: [CONFIG.R.thumbnail, CONFIG.R.oembed],
+                            width: oembed.thumbnail_width,
+                            height: data.height ? Math.round (oembed.thumbnail_width / (data.width / data.height)) : oembed.thumbnail_height
+                        }, {
+                            href: iframe.src,
+                            type: CONFIG.T.text_html,
+                            rel: [aspect > 1 ? CONFIG.R.player : CONFIG.R.reader, CONFIG.R.slideshow, CONFIG.R.html5],
+                            "aspect-ratio": aspect,
+                            "padding-bottom": 58
+                        }
+                    ]);
                 }
 
-                links.push ({
-                    href: oembed.thumbnail,
-                    type: CONFIG.T.image,
-                    rel: [CONFIG.R.thumbnail, CONFIG.R.oembed],
-                    width: oembed.thumbnail_width,
-                    height: data.height ? Math.round (oembed.thumbnail_width / (data.width / data.height)) : oembed.thumbnail_height
-                });
-
-                cb(null, links);
-
             });
         } else {
             cb(null, null);

From 2b689243a41a9c2652fc8bb079dbc7572a1acc26 Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Tue, 26 Oct 2021 15:25:19 -0400
Subject: [PATCH 062/102] Indent sample config

---
 config.local.js.SAMPLE | 596 ++++++++++++++++++++---------------------
 1 file changed, 298 insertions(+), 298 deletions(-)

diff --git a/config.local.js.SAMPLE b/config.local.js.SAMPLE
index d48e3e5e6..cf3c69de5 100644
--- a/config.local.js.SAMPLE
+++ b/config.local.js.SAMPLE
@@ -1,332 +1,332 @@
-    import { fileURLToPath } from 'url';
-    import { dirname } from 'path';
-    import { readFile } from 'fs/promises';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+import { readFile } from 'fs/promises';
 
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 
-    // Optional SSL cert, if you serve under HTTPS.
-    /*
-    const ssl_key = const file = await readFile(new URL('./key.pem', import.meta.url), {encoding: 'utf8'});
-    const ssl_cert = await readFile(new URL('./cert.pem', import.meta.url), {encoding: 'utf8'});
-    */
+// Optional SSL cert, if you serve under HTTPS.
+/*
+const ssl_key = const file = await readFile(new URL('./key.pem', import.meta.url), {encoding: 'utf8'});
+const ssl_cert = await readFile(new URL('./cert.pem', import.meta.url), {encoding: 'utf8'});
+*/
 
-    export default {
+export default {
 
-        // Specify a path for custom plugins. Custom plugins will override core plugins.
-        // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder',
+    // Specify a path for custom plugins. Custom plugins will override core plugins.
+    // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder',
 
-        DEBUG: false,
-        RICH_LOG_ENABLED: false,
+    DEBUG: false,
+    RICH_LOG_ENABLED: false,
 
-        // For embeds that require render, baseAppUrl will be used as the host.
-        baseAppUrl: "http://localhost:8061", // use "https://yourdomain.com/path" where you have Iframely in your reverse proxy
-        relativeStaticUrl: "/r",
+    // For embeds that require render, baseAppUrl will be used as the host.
+    baseAppUrl: "http://localhost:8061", // use "https://yourdomain.com/path" where you have Iframely in your reverse proxy
+    relativeStaticUrl: "/r",
 
-        // Or just skip built-in renders altogether
-        SKIP_IFRAMELY_RENDERS: true,
+    // Or just skip built-in renders altogether
+    SKIP_IFRAMELY_RENDERS: true,
 
-        // For legacy reasons the response format of Iframely open-source is
-        // different by default as it does not group the links array by rel.
-        // In order to get the same grouped response as in Cloud API,
-        // add `&group=true` to your request to change response per request
-        // or set `GROUP_LINKS` in your config to `true` for a global change.
-        GROUP_LINKS: true,
+    // For legacy reasons the response format of Iframely open-source is
+    // different by default as it does not group the links array by rel.
+    // In order to get the same grouped response as in Cloud API,
+    // add `&group=true` to your request to change response per request
+    // or set `GROUP_LINKS` in your config to `true` for a global change.
+    GROUP_LINKS: true,
 
-        // Number of maximum redirects to follow before aborting the page
-        // request with `redirect loop` error.
-        MAX_REDIRECTS: 4,
+    // Number of maximum redirects to follow before aborting the page
+    // request with `redirect loop` error.
+    MAX_REDIRECTS: 4,
 
-        SKIP_OEMBED_RE_LIST: [
-            // /^https?:\/\/yourdomain\.com\//,
-        ],
+    SKIP_OEMBED_RE_LIST: [
+        // /^https?:\/\/yourdomain\.com\//,
+    ],
 
-        /*
-        // Used to pass parameters to the generate functions when creating HTML elements
-        // disableSizeWrapper: Don't wrap element (iframe, video, etc) in a positioned div
-        GENERATE_LINK_PARAMS: {
-            disableSizeWrapper: true
-        },
-        */
+    /*
+    // Used to pass parameters to the generate functions when creating HTML elements
+    // disableSizeWrapper: Don't wrap element (iframe, video, etc) in a positioned div
+    GENERATE_LINK_PARAMS: {
+        disableSizeWrapper: true
+    },
+    */
 
-        port: 8061, //can be overridden by PORT env var
-        host: '0.0.0.0',    // Dockers beware. See https://github.com/itteco/iframely/issues/132#issuecomment-242991246
-                            //can be overridden by HOST env var
+    port: 8061, //can be overridden by PORT env var
+    host: '0.0.0.0',    // Dockers beware. See https://github.com/itteco/iframely/issues/132#issuecomment-242991246
+                        //can be overridden by HOST env var
 
-        // Optional SSL cert, if you serve under HTTPS.
-        /*
-        ssl: {
-            key: ssl_key,
-            cert: ssl_cert,
-            port: 443
-        },
-        */
+    // Optional SSL cert, if you serve under HTTPS.
+    /*
+    ssl: {
+        key: ssl_key,
+        cert: ssl_cert,
+        port: 443
+    },
+    */
 
-        /*
-        Supported cache engines:
-        - no-cache - no caching will be used.
-        - node-cache - good for debug, node memory will be used (https://github.com/tcs-de/nodecache).
-        - redis - https://github.com/mranney/node_redis.
-        - memcached - https://github.com/3rd-Eden/node-memcached
-        */
-        CACHE_ENGINE: 'node-cache',
-        CACHE_TTL: 0, // In seconds. 
-        // 0 = 'never expire' for memcached & node-cache to let cache engine decide itself when to evict the record
-        // 0 = 'no cache' for redis. Use high enough (e.g. 365*24*60*60*1000) ttl for similar 'never expire' approach instead
+    /*
+    Supported cache engines:
+    - no-cache - no caching will be used.
+    - node-cache - good for debug, node memory will be used (https://github.com/tcs-de/nodecache).
+    - redis - https://github.com/mranney/node_redis.
+    - memcached - https://github.com/3rd-Eden/node-memcached
+    */
+    CACHE_ENGINE: 'node-cache',
+    CACHE_TTL: 0, // In seconds. 
+    // 0 = 'never expire' for memcached & node-cache to let cache engine decide itself when to evict the record
+    // 0 = 'no cache' for redis. Use high enough (e.g. 365*24*60*60*1000) ttl for similar 'never expire' approach instead
 
-        /*
-        // Redis mode (cluster or standard)
-        REDIS_MODE: 'standard',
-        */
+    /*
+    // Redis mode (cluster or standard)
+    REDIS_MODE: 'standard',
+    */
 
-        /*
-        // Redis cache options.
-        REDIS_OPTIONS: {
-            host: '127.0.0.1',
-            port: 6379
-        },
-        */
+    /*
+    // Redis cache options.
+    REDIS_OPTIONS: {
+        host: '127.0.0.1',
+        port: 6379
+    },
+    */
 
-        /*
-        // Redis cluster options.
-        REDIS_CLUSTER_OPTIONS: {
-            servers: [
-                 {
-                     host: '10.0.0.10',
-                     port: 6379
-                 },
-                 // ...
-             ],
-        },
-        */
+    /*
+    // Redis cluster options.
+    REDIS_CLUSTER_OPTIONS: {
+        servers: [
+             {
+                 host: '10.0.0.10',
+                 port: 6379
+             },
+             // ...
+         ],
+    },
+    */
 
-        /*
-        // Memcached options. See https://github.com/3rd-Eden/node-memcached#server-locations
-        MEMCACHED_OPTIONS: {
-            locations: "127.0.0.1:11211"
-        }
-        */
+    /*
+    // Memcached options. See https://github.com/3rd-Eden/node-memcached#server-locations
+    MEMCACHED_OPTIONS: {
+        locations: "127.0.0.1:11211"
+    }
+    */
 
-        /*
-        // Access-Control-Allow-Origin list.
-        allowedOrigins: [
-            "*",
-            "http://another_domain.com"
+    /*
+    // Access-Control-Allow-Origin list.
+    allowedOrigins: [
+        "*",
+        "http://another_domain.com"
+    ],
+    */
+
+    /*
+    // Uncomment to enable plugin testing framework.
+    tests: {
+        mongodb: 'mongodb://localhost:27017/iframely-tests',
+        single_test_timeout: 10 * 1000,
+        plugin_test_period: 2 * 60 * 60 * 1000,
+        relaunch_script_period: 5 * 60 * 1000
+    },
+    */
+
+    // If there's no response from remote server, the timeout will occur after
+    RESPONSE_TIMEOUT: 5 * 1000, //ms
+
+    // Customize API calls to oembed endpoints.
+    // Must have: please add your `access_token` for Facebook and Instagram API calls
+    ADD_OEMBED_PARAMS: [{
+        
+        re: [ // Endpoint's URL regexp array.
+            /^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/instagram_oembed/i
         ],
-        */
+        params: { // Custom query-string params object.
 
-        /*
-        // Uncomment to enable plugin testing framework.
-        tests: {
-            mongodb: 'mongodb://localhost:27017/iframely-tests',
-            single_test_timeout: 10 * 1000,
-            plugin_test_period: 2 * 60 * 60 * 1000,
-            relaunch_script_period: 5 * 60 * 1000
+            // TODO: get your access Insagtam token as described 
+            // on https://developers.facebook.com/docs/instagram/oembed/                
+            access_token: '',   // The simplest way is 
+                                // to use `{app-id}|{app secret}` as access token
+
+            // Add any other optional params
+            hidecaption: true
+        }
+    }, {
+        re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_page/i],
+        params: {
+            // TODO: get your access token as described 
+            // on https://developers.facebook.com/docs/plugins/oembed                
+            access_token: '',   // The simplest way is 
+                                // to use `{app-id}|{app secret}` as access token
+
+            // Add any other optional params
+            show_posts: 0,
+            show_facepile: 0,
+            maxwidth: 600
+        }
+    }, {
+        // match i=user or i=moment or i=timeline to configure these types invidually
+        // see params spec at https://dev.twitter.com/web/embedded-timelines/oembed
+        re: [/^https?:\/\/publish\.twitter\.com\/oembed\?i=user/i],
+        params: {
+            limit: 1,
+            maxwidth: 600
+        }
+    }, {
+        // Facebook https://developers.facebook.com/docs/plugins/oembed/
+        re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_/i],
+        params: {
+            // TODO: get your access token as described 
+            // on https://developers.facebook.com/docs/plugins/oembed                
+            access_token: '',   // The simplest way is 
+                                // to use `{app-id}|{app secret}` as access token
+
+            // Add any other optional params, like skip script tag and fb-root div
+            // omitscript: true
+        }
+     }],
+
+    /* Configure use of HTTP proxies as needed. 
+       You don't have to specify all options per regex - just what you need to override
+    */
+    /*
+    PROXY: [{
+        re: [/^https?:\/\/www\.domain\.com/],
+        proxy_server: 'http://1.2.3.4:8080',
+        user_agent: 'CHANGE YOUR AGENT',
+        headers: {
+            // HTTP headers
+            // Overrides previous params if overlapped.
         },
-        */
+        request_options: {
+            // Refer to: https://github.com/request/request
+            // Overrides previous params if overlapped.
+        },
+        cache_ttl: 3600  // in seconds, cache response for 1 hour.
+    }],
+    */
 
-        // If there's no response from remote server, the timeout will occur after
-        RESPONSE_TIMEOUT: 5 * 1000, //ms
-
-        // Customize API calls to oembed endpoints.
-        // Must have: please add your `access_token` for Facebook and Instagram API calls
-        ADD_OEMBED_PARAMS: [{
-            
-            re: [ // Endpoint's URL regexp array.
-                /^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/instagram_oembed/i
-            ],
-            params: { // Custom query-string params object.
-
-                // TODO: get your access Insagtam token as described 
-                // on https://developers.facebook.com/docs/instagram/oembed/                
-                access_token: '',   // The simplest way is 
-                                    // to use `{app-id}|{app secret}` as access token
-
-                // Add any other optional params
-                hidecaption: true
-            }
-        }, {
-            re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_page/i],
-            params: {
-                // TODO: get your access token as described 
-                // on https://developers.facebook.com/docs/plugins/oembed                
-                access_token: '',   // The simplest way is 
-                                    // to use `{app-id}|{app secret}` as access token
-
-                // Add any other optional params
-                show_posts: 0,
-                show_facepile: 0,
-                maxwidth: 600
-            }
-        }, {
-            // match i=user or i=moment or i=timeline to configure these types invidually
-            // see params spec at https://dev.twitter.com/web/embedded-timelines/oembed
-            re: [/^https?:\/\/publish\.twitter\.com\/oembed\?i=user/i],
-            params: {
-                limit: 1,
-                maxwidth: 600
-            }
-        }, {
-            // Facebook https://developers.facebook.com/docs/plugins/oembed/
-            re: [/^https:\/\/graph\.facebook\.com\/v[0-9\.]+\/oembed_/i],
-            params: {
-                // TODO: get your access token as described 
-                // on https://developers.facebook.com/docs/plugins/oembed                
-                access_token: '',   // The simplest way is 
-                                    // to use `{app-id}|{app secret}` as access token
-
-                // Add any other optional params, like skip script tag and fb-root div
-                // omitscript: true
-            }
-         }],
+    // Customize API calls to 3rd parties. At the very least - configure required keys.
+    // For available provider options - please see the code of its domain plugin.
+    providerOptions: {
+        locale: "en_US",    // ISO 639-1 two-letter language code, e.g. en_CA or fr_CH. 
+                            // Will be added as highest priotity in accept-language header with each request. 
+                            // Plus is used in FB, YouTube and perhaps other plugins
+        "twitter": {
+            "max-width": 550,
+            "min-width": 250,
+            hide_media: false,
+            hide_thread: false,
+            omit_script: false,
+            center: false,
+            // dnt: true,                
+            cache_ttl: 100 * 365 * 24 * 3600 // 100 Years.
+        },
+        readability: {
+            enabled: false
+            // allowPTagDescription: true  // to enable description fallback to first paragraph
+        },
+        images: {
+            loadSize: false, // if true, will try an load first bytes of all images to get/confirm the sizes
+            checkFavicon: false // if true, will verify all favicons
+        },
+        tumblr: {
+            consumer_key: "INSERT YOUR VALUE"
+            // media_only: true     // disables status embeds for images and videos - will return plain media
+        },
+        google: {
+            // https://developers.google.com/maps/documentation/embed/guide#api_key
+            maps_key: "INSERT YOUR VALUE"
+        }, 
 
-        /* Configure use of HTTP proxies as needed. 
-           You don't have to specify all options per regex - just what you need to override
-        */
         /*
-        PROXY: [{
-            re: [/^https?:\/\/www\.domain\.com/],
-            proxy_server: 'http://1.2.3.4:8080',
-            user_agent: 'CHANGE YOUR AGENT',
-            headers: {
-                // HTTP headers
-                // Overrides previous params if overlapped.
-            },
-            request_options: {
-                // Refer to: https://github.com/request/request
-                // Overrides previous params if overlapped.
-            },
-            cache_ttl: 3600  // in seconds, cache response for 1 hour.
-        }],
+        // Optional Camo Proxy to wrap all images: https://github.com/atmos/camo
+        camoProxy: {
+            camo_proxy_key: "INSERT YOUR VALUE",
+            camo_proxy_host: "INSERT YOUR VALUE"
+            // ssl_only: true // will only proxy non-ssl images
+        },
         */
 
-        // Customize API calls to 3rd parties. At the very least - configure required keys.
-        // For available provider options - please see the code of its domain plugin.
-        providerOptions: {
-            locale: "en_US",    // ISO 639-1 two-letter language code, e.g. en_CA or fr_CH. 
-                                // Will be added as highest priotity in accept-language header with each request. 
-                                // Plus is used in FB, YouTube and perhaps other plugins
-            "twitter": {
-                "max-width": 550,
-                "min-width": 250,
-                hide_media: false,
-                hide_thread: false,
-                omit_script: false,
-                center: false,
-                // dnt: true,                
-                cache_ttl: 100 * 365 * 24 * 3600 // 100 Years.
-            },
-            readability: {
-                enabled: false
-                // allowPTagDescription: true  // to enable description fallback to first paragraph
-            },
-            images: {
-                loadSize: false, // if true, will try an load first bytes of all images to get/confirm the sizes
-                checkFavicon: false // if true, will verify all favicons
-            },
-            tumblr: {
-                consumer_key: "INSERT YOUR VALUE"
-                // media_only: true     // disables status embeds for images and videos - will return plain media
-            },
-            google: {
-                // https://developers.google.com/maps/documentation/embed/guide#api_key
-                maps_key: "INSERT YOUR VALUE"
-            }, 
-
-            /*
-            // Optional Camo Proxy to wrap all images: https://github.com/atmos/camo
-            camoProxy: {
-                camo_proxy_key: "INSERT YOUR VALUE",
-                camo_proxy_host: "INSERT YOUR VALUE"
-                // ssl_only: true // will only proxy non-ssl images
-            },
-            */
-
-            // List of query parameters to add to YouTube and Vimeo frames
-            // Start it with leading "?". Or omit alltogether for default values
-            // API key is optional, youtube will work without it too.
-            // It is probably the same API key you use for Google Maps.
-            youtube: {
-                // api_key: "INSERT YOUR VALUE",
-                // parts: [ "snippet", "player" ], // list of fields you want to use in the request, in most cases you only need those two
-                get_params: "?rel=0&showinfo=1"     // https://developers.google.com/youtube/player_parameters
-            },
-            vimeo: {
-                get_params: "?byline=0&badge=0"     // https://developer.vimeo.com/player/embedding
-            },
-            soundcloud: {
-                old_player: true // enables classic player
-            },
-            giphy: {
-                media_only: true // disables branded player for gifs and returns just the image
-            },
-            bandcamp: {
-                get_params: '/size=large/bgcol=333333/linkcol=ffffff/artwork=small/transparent=true/',
-                media: {
-                    album: {
-                        height: 472,
-                        'max-width': 700
-                    },
-                    track: {
-                        height: 120,
-                        'max-width': 700
-                    }
+        // List of query parameters to add to YouTube and Vimeo frames
+        // Start it with leading "?". Or omit alltogether for default values
+        // API key is optional, youtube will work without it too.
+        // It is probably the same API key you use for Google Maps.
+        youtube: {
+            // api_key: "INSERT YOUR VALUE",
+            // parts: [ "snippet", "player" ], // list of fields you want to use in the request, in most cases you only need those two
+            get_params: "?rel=0&showinfo=1"     // https://developers.google.com/youtube/player_parameters
+        },
+        vimeo: {
+            get_params: "?byline=0&badge=0"     // https://developer.vimeo.com/player/embedding
+        },
+        soundcloud: {
+            old_player: true // enables classic player
+        },
+        giphy: {
+            media_only: true // disables branded player for gifs and returns just the image
+        },
+        bandcamp: {
+            get_params: '/size=large/bgcol=333333/linkcol=ffffff/artwork=small/transparent=true/',
+            media: {
+                album: {
+                    height: 472,
+                    'max-width': 700
+                },
+                track: {
+                    height: 120,
+                    'max-width': 700
                 }
-            },
-            // Docs: https://dev.twitch.tv/docs/embed/video-and-clips
-            /*
-            twitch: {
-                parent: 'jsbin.com, null.jsbin.com, localhost'
-            },
-            */
+            }
         },
-
-        // WHITELIST_WILDCARD, if present, will be added to whitelist as record for top level domain: "*"
-        // with it, you can define what parsers do when they run accross unknown publisher.
-        // If absent or empty, all generic media parsers will be disabled except for known domains
-        // More about format: https://iframely.com/docs/qa-format
-
+        // Docs: https://dev.twitch.tv/docs/embed/video-and-clips
         /*
-        WHITELIST_WILDCARD: {
-              "twitter": {
-                "player": "allow",
-                "photo": "deny"
-              },
-              "oembed": {
-                "video": "allow",
-                "photo": "allow",
-                "rich": "deny",
-                "link": "deny"
-              },
-              "og": {
-                "video": ["allow", "ssl", "responsive"]
-              },
-              "iframely": {
-                "survey": "allow",
-                "reader": "allow",
-                "player": "allow",
-                "image": "allow"
-              },
-              "html-meta": {
-                "video": ["allow", "responsive"],
-                "promo": "allow"
-              }
-        }
+        twitch: {
+            parent: 'jsbin.com, null.jsbin.com, localhost'
+        },
         */
+    },
 
-        // The list of regexs to be ignored. Iframely will return 417
-        // At minimum, keep your localhosts blacklisted to avoid SSRF
-        IGNORE_DOMAINS_RE: [
-            /^https?:\/\/127\.0\.0\.1/i,
-            /^https?:\/\/localhost/i,
-            /^https?:\/\/[^\/]+:\d+\/?/, // Blocks port-scan via DNS pointing to 127.0.0.1
+    // WHITELIST_WILDCARD, if present, will be added to whitelist as record for top level domain: "*"
+    // with it, you can define what parsers do when they run accross unknown publisher.
+    // If absent or empty, all generic media parsers will be disabled except for known domains
+    // More about format: https://iframely.com/docs/qa-format
 
-            // And this is AWS metadata service
-            // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
-            /^https?:\/\/169\.254\.169\.254/
-        ],
+    /*
+    WHITELIST_WILDCARD: {
+          "twitter": {
+            "player": "allow",
+            "photo": "deny"
+          },
+          "oembed": {
+            "video": "allow",
+            "photo": "allow",
+            "rich": "deny",
+            "link": "deny"
+          },
+          "og": {
+            "video": ["allow", "ssl", "responsive"]
+          },
+          "iframely": {
+            "survey": "allow",
+            "reader": "allow",
+            "player": "allow",
+            "image": "allow"
+          },
+          "html-meta": {
+            "video": ["allow", "responsive"],
+            "promo": "allow"
+          }
+    }
+    */
 
-        // Endpoint for prerender service, if you need it. Used to parse React apps. Very slow.
-        // Tested with https://github.com/prerender/prerender
-        // PRERENDER_URL: "https://domain/render?url="
-    };
\ No newline at end of file
+    // The list of regexs to be ignored. Iframely will return 417
+    // At minimum, keep your localhosts blacklisted to avoid SSRF
+    IGNORE_DOMAINS_RE: [
+        /^https?:\/\/127\.0\.0\.1/i,
+        /^https?:\/\/localhost/i,
+        /^https?:\/\/[^\/]+:\d+\/?/, // Blocks port-scan via DNS pointing to 127.0.0.1
+
+        // And this is AWS metadata service
+        // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+        /^https?:\/\/169\.254\.169\.254/
+    ],
+
+    // Endpoint for prerender service, if you need it. Used to parse React apps. Very slow.
+    // Tested with https://github.com/prerender/prerender
+    // PRERENDER_URL: "https://domain/render?url="
+};
\ No newline at end of file

From b325e3328e05e55f6875b6f19f1e46f75f1ad234 Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Tue, 26 Oct 2021 21:12:57 -0400
Subject: [PATCH 063/102] twitter: remove now obsolete 1.1 oauth APi

---
 plugins/domains/twitter.com/twitter.status.js | 95 +++----------------
 1 file changed, 11 insertions(+), 84 deletions(-)

diff --git a/plugins/domains/twitter.com/twitter.status.js b/plugins/domains/twitter.com/twitter.status.js
index 21abc6435..c74a970cb 100644
--- a/plugins/domains/twitter.com/twitter.status.js
+++ b/plugins/domains/twitter.com/twitter.status.js
@@ -15,66 +15,27 @@ export default {
     mixins: ['domain-icon'],
 
     getData: function(urlMatch, request, options, cb) {
-        var id = urlMatch[1];
 
         var c = options.getProviderOptions("twitter") || options.getProviderOptions("twitter.status");
 
         if (c.disabled) {
-            return cb('Twitter API Disabled');
+            return cb('Twitter API disabled');
         }
 
-        var oauth = c.consumer_key 
-            ? {
-                consumer_key: c.consumer_key,
-                consumer_secret: c.consumer_secret,
-                token: c.access_token,
-                token_secret: c.access_token_secret
-            } : false;
-
-        var blockExpireIn = 0;
-        var block_key = 'twbl:' + c.consumer_key;
-
         async.waterfall([
 
             function(cb) {
 
-                if (oauth) {
-                    cache.get(block_key, cb);
-                } else {
-                    cb(null, null);
-                }
-            },
-
-            function(expireIn, cb) {
-
-                if (expireIn) {
-                    var now = Math.round(new Date().getTime() / 1000);
-                    if (expireIn > now) {
-                        blockExpireIn = expireIn - now;
-                    }
-                }
-
-                var usePublicApi = !oauth || blockExpireIn > 0;
-
-                var apiUrl;
-
-                var qs = {
-                    hide_media:  c.hide_media, 
-                    hide_thread: true, //  c.hide_thread - now handled in getLinks. This is the only reliable way to detect if a tweet has the thread
-                    omit_script: c.omit_script
-                };
+                var id = urlMatch[1];
 
-                if (usePublicApi) {
-                    apiUrl = "https://publish.twitter.com/oembed";
-                    qs.url = urlMatch[0];
-                } else {
-                    apiUrl = "https://api.twitter.com/1.1/statuses/oembed.json";
-                    qs.id = id;
-                }
-
-                request(_.extend({
-                    url: apiUrl,
-                    qs: qs,
+                request({
+                    url: "https://publish.twitter.com/oembed",
+                    qs: {
+                        hide_media:  c.hide_media, 
+                        hide_thread: true, //  c.hide_thread - now handled in getLinks. This is the only reliable way to detect if a tweet has the thread
+                        omit_script: c.omit_script,
+                        url: urlMatch[0]
+                    },
                     json: true,
                     cache_key: 'twitter:oembed:' + id,
                     prepareResult: function(error, response, data, cb) {
@@ -91,40 +52,6 @@ export default {
                             }
                         }
 
-                        // Do not block 1.1 api if data from cache.
-                        if (oauth && !response.fromRequestCache) {
-
-                            var remaining = parseInt(response.headers['x-rate-limit-remaining']);
-
-                            if (response.statusCode === 429 || remaining <= 7) {
-                                var now = Math.round(new Date().getTime() / 1000);
-                                var limitResetAt = parseInt(response.headers['x-rate-limit-reset']);
-                                var ttl = limitResetAt - now;
-
-                                // Do not allow ttl 0.
-                                // 5 seconds - to cover possible time difference with twitter.
-                                if (ttl < 5) {
-                                    ttl = 5;
-                                }
-
-                                // Block maximum for 15 minutes.
-                                if (ttl > 15*60) {
-                                    ttl = 15*60
-                                }
-
-                                if (response.statusCode === 429) {
-                                    log('   -- Twitter API limit reached by status code 429. Disabling for ' + ttl + ' seconds.');
-                                } else {
-                                    log('   -- Twitter API limit warning, remaining calls: ' + remaining + '. Disabling for ' + ttl + ' seconds.');
-                                }
-
-                                // Store expire date as value to be sure it past.
-                                var expireIn = now + ttl;
-
-                                cache.set(block_key, expireIn, {ttl: ttl});
-                            }
-                        }
-
                         if (response.statusCode === 404) {
                             return cb({
                                 responseStatusCode: 404,
@@ -147,7 +74,7 @@ export default {
 
                         cb(error, data);
                     }
-                }, usePublicApi ? null : {oauth: oauth}), cb); // add oauth if 1.1, else skip it
+                }, cb);
 
             }
 

From 85ad209ad24af083c4a71b8caf92545896c606de Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Tue, 26 Oct 2021 21:16:39 -0400
Subject: [PATCH 064/102] twitter - remove one more obsolete reference

---
 plugins/domains/twitter.com/twitter.status.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/plugins/domains/twitter.com/twitter.status.js b/plugins/domains/twitter.com/twitter.status.js
index c74a970cb..a76b3d2e9 100644
--- a/plugins/domains/twitter.com/twitter.status.js
+++ b/plugins/domains/twitter.com/twitter.status.js
@@ -45,11 +45,7 @@ export default {
                         }
 
                         if (response.fromRequestCache) {
-                            if (blockExpireIn > 0) {
-                                log('   -- Twitter API limit reached (' + blockExpireIn + ' seconds left), but cache used.');
-                            } else {
-                                log('   -- Twitter API cache used.');
-                            }
+                            log('   -- Twitter API cache used.');
                         }
 
                         if (response.statusCode === 404) {

From 5d93d5d32fc66a806b340e546d6f83c8e46b86fe Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Tue, 26 Oct 2021 21:25:52 -0400
Subject: [PATCH 065/102] twitter - clean up obsolete async

---
 plugins/domains/twitter.com/twitter.status.js | 142 ++++++++----------
 1 file changed, 59 insertions(+), 83 deletions(-)

diff --git a/plugins/domains/twitter.com/twitter.status.js b/plugins/domains/twitter.com/twitter.status.js
index a76b3d2e9..b5a7b8f50 100644
--- a/plugins/domains/twitter.com/twitter.status.js
+++ b/plugins/domains/twitter.com/twitter.status.js
@@ -1,14 +1,9 @@
-import * as async from 'async';
-import { cache } from '../../../lib/cache.js';
 import log from '../../../logging.js';
-import * as _ from 'underscore';
 import * as entities from 'entities';
 
 export default {
 
-    re: [
-        /^https?:\/\/twitter\.com\/(?:\w+)\/status(?:es)?\/(\d+)/i
-    ],
+    re: /^https?:\/\/twitter\.com\/(?:\w+)\/status(?:es)?\/(\d+)/i,
 
     provides: ['twitter_oembed', 'twitter_og', '__allowTwitterOg'],
 
@@ -22,84 +17,65 @@ export default {
             return cb('Twitter API disabled');
         }
 
-        async.waterfall([
-
-            function(cb) {
-
-                var id = urlMatch[1];
-
-                request({
-                    url: "https://publish.twitter.com/oembed",
-                    qs: {
-                        hide_media:  c.hide_media, 
-                        hide_thread: true, //  c.hide_thread - now handled in getLinks. This is the only reliable way to detect if a tweet has the thread
-                        omit_script: c.omit_script,
-                        url: urlMatch[0]
-                    },
-                    json: true,
-                    cache_key: 'twitter:oembed:' + id,
-                    prepareResult: function(error, response, data, cb) {
-
-                        if (error) {
-                            return cb(error);
-                        }
-
-                        if (response.fromRequestCache) {
-                            log('   -- Twitter API cache used.');
-                        }
-
-                        if (response.statusCode === 404) {
-                            return cb({
-                                responseStatusCode: 404,
-                                message: 'The tweet is no longer available.'
-                            })
-                        } else if (response.statusCode === 403) {
-                            return cb({
-                                responseStatusCode: 404,
-                                message: 'It looks this Twitter account has been suspended.'
-                            })
-
-                        } else if (response.statusCode !== 200) {
-                            return cb('Non-200 response from Twitter API (statuses/oembed.json: ' + response.statusCode);
-                        }
-
-                        if (typeof data !== 'object') {
-                            return cb('Object expected in Twitter API (statuses/oembed.json), got: ' + data);
-                        }
-
-
-                        cb(error, data);
-                    }
-                }, cb);
-
+        request({
+            url: "https://publish.twitter.com/oembed",
+            qs: {
+                hide_media:  c.hide_media, 
+                hide_thread: true, //  c.hide_thread - now handled in getLinks. This is the only reliable way to detect if a tweet has the thread
+                omit_script: c.omit_script,
+                url: urlMatch[0]
+            },
+            json: true,
+            cache_key: 'twitter:oembed:' + urlMatch[1],
+            prepareResult: function(error, response, oembed, cb) {
+
+                if (error) {
+                    return cb(error);
+                }
+
+                if (response.fromRequestCache) {
+                    log('   -- Twitter API cache used.');
+                }
+
+                if (response.statusCode === 404) {
+                    return cb({
+                        responseStatusCode: 404,
+                        message: 'The tweet is no longer available.'
+                    })
+
+                } else if (response.statusCode === 403) {
+                    return cb({
+                        responseStatusCode: 404,
+                        message: 'It looks this Twitter account has been suspended.'
+                    })
+
+                } else if (response.statusCode !== 200) {
+                    return cb('Non-200 response from Twitter API (statuses/oembed.json: ' + response.statusCode);
+                }
+
+                if (typeof oembed !== 'object') {
+                    return cb('Object expected in Twitter API (statuses/oembed.json), got: ' + oembed);
+                }
+
+                oembed.title = oembed.author_name + ' on Twitter';
+                oembed["min-width"] = c["min-width"];
+                oembed["max-width"] = c["max-width"];
+
+                var result = {
+                    twitter_oembed: oembed
+                };
+
+                if (/pic\.twitter\.com/i.test(oembed.html)) {
+                    result.__allowTwitterOg = true;
+                    options.followHTTPRedirect = true; // avoid core's re-directs. Use HTTP request redirects instead
+                    options.exposeStatusCode = true;
+                } else {
+                    result.twitter_og = false;
+                }
+
+                return cb(error, result);
             }
-
-        ], function(error, oembed) {
-
-
-            if (error) {
-                return cb(error);
-            }
-
-            oembed.title = oembed.author_name + ' on Twitter';
-
-            oembed["min-width"] = c["min-width"];
-            oembed["max-width"] = c["max-width"];
-
-            var result = {
-                twitter_oembed: oembed
-            };
-
-            if (/pic\.twitter\.com/i.test(oembed.html)) {
-                result.__allowTwitterOg = true;
-                options.followHTTPRedirect = true; // avoid core's re-directs. Use HTTP request redirects instead
-                options.exposeStatusCode = true;
-            } else {
-                result.twitter_og = false;
-            }
-
-            cb(null, result);
-        });
+        }, cb);
     },
 
     getMeta: function(twitter_oembed) {

From 475cfb6864af5d00b181072f69fd38222b82b371 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 27 Oct 2021 16:14:44 +0300
Subject: [PATCH 066/102] request: `json: false` prevents json parsing

---
 lib/fetch.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 4552ed742..a28cd363e 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -61,6 +61,7 @@ export function fetchStreamAuthorized(options) {
 
 export function fetchData(options) {
     var json = options.json;
+    delete options.json;
     var response;
     const fetch_options = Object.assign({}, options);
     const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
@@ -68,7 +69,10 @@ export function fetchData(options) {
         fetch(uri, fetch_options)
             .then(res => {
                 response = res;
-                json = json || (response.headers.get('content-type').indexOf('application/json') > -1);
+                if (json !== false) {
+                    // If `json` not forbidden, read `content-type`.
+                    json = json || (response.headers.get('content-type').indexOf('application/json') > -1);
+                }
                 if (json) {
                     return response.json();
                 } else {

From 8ab16b5fd37b1720233a24cc6808c3c3af44489d Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 27 Oct 2021 19:27:49 +0300
Subject: [PATCH 067/102] utils: allow use custom proxy

---
 lib/utils.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/utils.js b/lib/utils.js
index 8f613893e..c0845a3ea 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -81,8 +81,10 @@ export function prepareRequestOptions(request_options, options) {
         });
         if (proxy) {
             if (proxy.prerender && CONFIG.PRERENDER_URL) {
-                // `url` used above for agent check.
                 request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri);
+
+            } else if (proxy.proxy && CONFIG.PROXY_URL) {
+                request_options.uri = CONFIG.PROXY_URL + encodeURIComponent(uri);
             }
             if (proxy.proxy_server) {
                 request_options.proxy = proxy.proxy_server;

From b41a5ccc650d851580ffc6ef3d25c20169ed59f4 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 27 Oct 2021 22:03:44 +0300
Subject: [PATCH 068/102] fix using cheerio `$script.text()`

---
 plugins/domains/geogebra.iframe.js | 6 ++----
 plugins/links/embedURL/embedURL.js | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/plugins/domains/geogebra.iframe.js b/plugins/domains/geogebra.iframe.js
index c707a2fb7..119da366d 100644
--- a/plugins/domains/geogebra.iframe.js
+++ b/plugins/domains/geogebra.iframe.js
@@ -4,13 +4,11 @@ export default {
 
     // It's here mostly just to detect proper embed sizing.
     getLink: function(url, cheerio) {
-        var $el = cheerio('script');
-
         var $script = cheerio('script:contains("var parameters =")');
 
-        if ($script.length === 1 && /({.+});/i.test($script.text())) {
+        if ($script.length === 1 && /({.+});/i.test($script.html())) {
             try {
-                var params = JSON.parse ($script.text().match(/({.+});/i)[1]);
+                var params = JSON.parse ($script.html().match(/({.+});/i)[1]);
 
                 if (params.width && params.height) {
                     return {
diff --git a/plugins/links/embedURL/embedURL.js b/plugins/links/embedURL/embedURL.js
index 68ef34bf2..b341fe0e0 100644
--- a/plugins/links/embedURL/embedURL.js
+++ b/plugins/links/embedURL/embedURL.js
@@ -12,7 +12,7 @@ export default {
 
         if ($script.length === 1) {
             try {
-                var json = utils.parseJSONSource($script.text());
+                var json = utils.parseJSONSource($script.html());
 
                 if (json['@type']) {
                     ld = {};

From 957c755f7b05835de05b746dbcd9806c8a8a221c Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 27 Oct 2021 22:36:10 +0300
Subject: [PATCH 069/102] fetch: fix default follow redirect if option is not
 specified

---
 lib/utils.js | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/lib/utils.js b/lib/utils.js
index c0845a3ea..50ba1f68b 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -128,6 +128,16 @@ export function prepareRequestOptions(request_options, options) {
 
     var options = options || {};
 
+    var redirect, follow;
+    if (options.followRedirect) {
+        redirect = 'follow';
+        follow = options.maxRedirects || CONFIG.MAX_REDIRECTS;
+    }
+    if (options.followRedirect === false) {
+        redirect = 'manual';
+        follow = 0;
+    }
+
     var request_options = prepareRequestOptions({
         // TODO: jar: jar,
         
@@ -139,8 +149,8 @@ export function prepareRequestOptions(request_options, options) {
             'Accept': '*/*'
         },
         timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        redirect: options.followRedirect ? 'follow' : 'manual',
-        follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
+        redirect: redirect,
+        follow: follow
     }, options);
 
     try {

From 921915ac06c91af012cac891d06ff758f899751b Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 27 Oct 2021 22:39:09 +0300
Subject: [PATCH 070/102] fetch: fix default follow redirect if option is not
 specified

---
 lib/utils.js | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/utils.js b/lib/utils.js
index 50ba1f68b..644e9d539 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -184,7 +184,16 @@ var getHead = function(url, options, callbacks) {
     //     jar = request.jar();
     // }
 
-    // TODO: review node-fetch options
+    var redirect, follow;
+    if (options.followRedirect) {
+        redirect = 'follow';
+        follow = options.maxRedirects || CONFIG.MAX_REDIRECTS;
+    }
+    if (options.followRedirect === false) {
+        redirect = 'manual';
+        follow = 0;
+    }
+    
     var request_options = prepareRequestOptions({
         // jar: jar,
 
@@ -195,8 +204,8 @@ var getHead = function(url, options, callbacks) {
             'User-Agent': CONFIG.USER_AGENT
         },
         timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT,
-        redirect: options.followRedirect ? 'follow' : 'manual',
-        follow: options.followRedirect ? (options.maxRedirects || CONFIG.MAX_REDIRECTS) : 0
+        redirect: redirect,
+        follow: follow
         // No abort controller for head.
     });
 

From 9500e14da1f3592cf22a65ad57c601731a41f65b Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Wed, 27 Oct 2021 15:48:28 -0400
Subject: [PATCH 071/102] tests: fix ld-json

---
 plugins/links/embedURL/embedURL.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/links/embedURL/embedURL.js b/plugins/links/embedURL/embedURL.js
index b341fe0e0..66a03de4e 100644
--- a/plugins/links/embedURL/embedURL.js
+++ b/plugins/links/embedURL/embedURL.js
@@ -15,7 +15,7 @@ export default {
                 var json = utils.parseJSONSource($script.html());
 
                 if (json['@type']) {
-                    ld = {};
+                    var ld = {};
                     ld[json['@type'].toLowerCase()] = json;
 
                     if (__allowEmbedURL !== 'skip_ld') {

From 5322befdeccff3b71594485c6804a679f3ba4628 Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Thu, 28 Oct 2021 10:19:58 -0400
Subject: [PATCH 072/102] tests: fix scribd

---
 plugins/domains/scribd.com/scribd.com.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/domains/scribd.com/scribd.com.js b/plugins/domains/scribd.com/scribd.com.js
index f2d24749c..fd53f1be1 100644
--- a/plugins/domains/scribd.com/scribd.com.js
+++ b/plugins/domains/scribd.com/scribd.com.js
@@ -54,7 +54,8 @@ export default {
     },
 
     tests: [{
-        noFeeds: true
+        noFeeds: true,
+        skipMethods: ['getData']
     },
         "https://www.scribd.com/doc/116154615/Australia-Council-Arts-Funding-Guide-2013",
         "https://www.scribd.com/document/399637688/Prestons-Advert"

From 9c57803bb880968d748862e9fa13294dc0804fc9 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 28 Oct 2021 17:33:47 +0300
Subject: [PATCH 073/102] try fix `ABORT_ERR`

---
 lib/plugins/system/htmlparser/htmlparser.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 0dd15e4dd..1789f6ab2 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -85,7 +85,8 @@ export default {
                 const abortController = resp.abortController;
 
                 if (resp.status >= 300 && resp.status< 400 && headers.location) {
-                    abortController.abort();
+                    // abortController.abort();
+                    resp.pause();
                     var redirectUrl = urlLib.resolve(url, headers.location);
                     // Prevent cache self redirect. Some sites changes cookies and stops redirect loop (e.g. https://miro.com/app/live-embed/o9J_lBwNMhI=/?embedAutoplay=true)
                     var preventCache = redirectUrl === url;
@@ -101,7 +102,8 @@ export default {
                 }
 
                 if (resp.status !== 200) {
-                    abortController.abort();
+                    // abortController.abort();
+                    resp.pause();
                     if (!!options.exposeStatusCode) {
                         return cacheAndRespond(null, {
                             __statusCode: resp.status,

From edfd472134e48402534e5be3d695dd1991ee99b6 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 28 Oct 2021 18:09:08 +0300
Subject: [PATCH 074/102] fetch: allow disable http2

---
 lib/fetch.js | 28 +++++++++++++++++++++-------
 lib/utils.js |  3 +++
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index a28cd363e..3542893dc 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,4 +1,4 @@
-import { noCache, createUrl, AbortController, AbortError } from '@adobe/helix-fetch';
+import { h1NoCache, noCache, createUrl, AbortController, AbortError } from '@adobe/helix-fetch';
 
 const fetchKeepAlive = noCache({
     h1: {
@@ -7,13 +7,25 @@ const fetchKeepAlive = noCache({
     rejectUnauthorized: false   // By default skip auth check for all.
 }).fetch;
 
+const fetchH1KeepAlive = h1NoCache({
+    h1: {
+        keepAlive: true
+    },
+    rejectUnauthorized: false   // By default skip auth check for all.
+}).fetch;
+
 const fetchAuthorized = noCache().fetch;    // `rejectUnauthorized: true` - by `fetch` default.
+const fetchH1Authorized = h1NoCache().fetch;      // `rejectUnauthorized: true` - by `fetch` default.
 
 const { fetch } = noCache({
     rejectUnauthorized: false   // By default skip auth check for all.
 });
 
-function doFetch(fetch_func, options) {
+const fetchH1 = h1NoCache({
+    rejectUnauthorized: false   // By default skip auth check for all.
+}).fetch;
+
+function doFetch(fetch_func, h1_fetch_func, options) {
     const abortController = new AbortController();
     const fetch_options = Object.assign({}, options);
     // Allow request abort before finish.
@@ -24,8 +36,9 @@ function doFetch(fetch_func, options) {
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
     // Implement `qs` (get params).
     const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
+    const a_fetch_func = options.disable_http2 ? h1_fetch_func: fetch_func;
     return new Promise((resolve, reject) => {
-        fetch_func(uri, fetch_options)
+        a_fetch_func(uri, fetch_options)
             .then(response => {
                 var stream = response.body;
                 stream.status = response.status;
@@ -48,15 +61,15 @@ function doFetch(fetch_func, options) {
 }
 
 export function fetchStreamKeepAlive(options) {
-    return doFetch(fetchKeepAlive, options);
+    return doFetch(fetchKeepAlive, fetchH1KeepAlive, options);
 }
 
 export function fetchStream(options) {
-    return doFetch(fetch, options);
+    return doFetch(fetch, fetchH1, options);
 };
 
 export function fetchStreamAuthorized(options) {
-    return doFetch(fetchAuthorized, options);
+    return doFetch(fetchAuthorized, fetchH1Authorized, options);
 };
 
 export function fetchData(options) {
@@ -65,8 +78,9 @@ export function fetchData(options) {
     var response;
     const fetch_options = Object.assign({}, options);
     const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
+    const a_fetch_func = options.disable_http2 ? fetchH1: fetch;
     return new Promise((resolve, reject) => {
-        fetch(uri, fetch_options)
+        a_fetch_func(uri, fetch_options)
             .then(res => {
                 response = res;
                 if (json !== false) {
diff --git a/lib/utils.js b/lib/utils.js
index 644e9d539..1ea987944 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -100,6 +100,9 @@ export function prepareRequestOptions(request_options, options) {
             if (proxy.request_options) {
                 _.extend(request_options, proxy.request_options);
             }
+            if (proxy.disable_http2) {
+                request_options.disable_http2 = true;
+            }
         }
     }
 

From 574bc46138c1dceaab2fc471a30108b42ea50d10 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 28 Oct 2021 20:24:35 +0300
Subject: [PATCH 075/102] use `cheerio` 0.22

---
 package.json |   2 +-
 yarn.lock    | 250 ++++++++++++++++++++++++++++++++++++---------------
 2 files changed, 177 insertions(+), 75 deletions(-)

diff --git a/package.json b/package.json
index 39eca5bce..554395e6d 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,7 @@
     "dependencies": {
         "@adobe/helix-fetch": "^3.0.0",
         "async": "2.4.1",
-        "cheerio": "^1.0.0-rc.10",
+        "cheerio": "^0.22.0",
         "chokidar": "^3.3.1",
         "ejs": "^3.1.6",
         "entities": "1.1.1",
diff --git a/yarn.lock b/yarn.lock
index e1a905870..df3e02b76 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -212,7 +212,7 @@ body-parser@1.19.0, body-parser@^1.18.1:
     raw-body "2.4.0"
     type-is "~1.6.17"
 
-boolbase@^1.0.0:
+boolbase@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
   integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24=
@@ -290,29 +290,27 @@ chalk@^2.4.2:
     escape-string-regexp "^1.0.5"
     supports-color "^5.3.0"
 
-cheerio-select@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/cheerio-select/-/cheerio-select-1.5.0.tgz#faf3daeb31b17c5e1a9dabcee288aaf8aafa5823"
-  integrity sha512-qocaHPv5ypefh6YNxvnbABM07KMxExbtbfuJoIie3iZXX1ERwYmJcIiRrr9H05ucQP1k28dav8rpdDgjQd8drg==
-  dependencies:
-    css-select "^4.1.3"
-    css-what "^5.0.1"
-    domelementtype "^2.2.0"
-    domhandler "^4.2.0"
-    domutils "^2.7.0"
-
-cheerio@^1.0.0-rc.10:
-  version "1.0.0-rc.10"
-  resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.10.tgz#2ba3dcdfcc26e7956fc1f440e61d51c643379f3e"
-  integrity sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==
-  dependencies:
-    cheerio-select "^1.5.0"
-    dom-serializer "^1.3.2"
-    domhandler "^4.2.0"
-    htmlparser2 "^6.1.0"
-    parse5 "^6.0.1"
-    parse5-htmlparser2-tree-adapter "^6.0.1"
-    tslib "^2.2.0"
+cheerio@^0.22.0:
+  version "0.22.0"
+  resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-0.22.0.tgz#a9baa860a3f9b595a6b81b1a86873121ed3a269e"
+  integrity sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=
+  dependencies:
+    css-select "~1.2.0"
+    dom-serializer "~0.1.0"
+    entities "~1.1.1"
+    htmlparser2 "^3.9.1"
+    lodash.assignin "^4.0.9"
+    lodash.bind "^4.1.4"
+    lodash.defaults "^4.0.1"
+    lodash.filter "^4.4.0"
+    lodash.flatten "^4.2.0"
+    lodash.foreach "^4.3.0"
+    lodash.map "^4.4.0"
+    lodash.merge "^4.4.0"
+    lodash.pick "^4.2.1"
+    lodash.reduce "^4.4.0"
+    lodash.reject "^4.4.0"
+    lodash.some "^4.4.0"
 
 chokidar@^3.3.1:
   version "3.5.2"
@@ -420,21 +418,20 @@ core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"
   integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==
 
-css-select@^4.1.3:
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.1.3.tgz#a70440f70317f2669118ad74ff105e65849c7067"
-  integrity sha512-gT3wBNd9Nj49rAbmtFHj1cljIAOLYSX1nZ8CB7TBO3INYckygm5B7LISU/szY//YmdiSLbJvDLOx9VnMVpMBxA==
+css-select@~1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
+  integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=
   dependencies:
-    boolbase "^1.0.0"
-    css-what "^5.0.0"
-    domhandler "^4.2.0"
-    domutils "^2.6.0"
-    nth-check "^2.0.0"
+    boolbase "~1.0.0"
+    css-what "2.1"
+    domutils "1.5.1"
+    nth-check "~1.0.1"
 
-css-what@^5.0.0, css-what@^5.0.1:
-  version "5.1.0"
-  resolved "https://registry.yarnpkg.com/css-what/-/css-what-5.1.0.tgz#3f7b707aadf633baf62c2ceb8579b545bb40f7fe"
-  integrity sha512-arSMRWIIFY0hV8pIxZMEfmMI47Wj3R/aWpZDDxWYCPEiOMv6tfOrnpDtgxBYPEQD4V0Y/958+1TdC3iWTFcUPw==
+css-what@2.1:
+  version "2.1.3"
+  resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.3.tgz#a6d7604573365fe74686c3f311c56513d88285f2"
+  integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==
 
 dashdash@^1.12.0:
   version "1.14.1"
@@ -520,7 +517,15 @@ dir-glob@^3.0.1:
   dependencies:
     path-type "^4.0.0"
 
-dom-serializer@^1.0.1, dom-serializer@^1.3.2:
+dom-serializer@0:
+  version "0.2.2"
+  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51"
+  integrity sha512-2/xPb3ORsQ42nHYiSunXkDjPLBaEj/xTwUO4B7XCZQTRk7EBtTOPaygh10YAAh2OI1Qrp6NWfpAhzswj0ydt9g==
+  dependencies:
+    domelementtype "^2.0.1"
+    entities "^2.0.0"
+
+dom-serializer@^1.0.1:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.3.2.tgz#6206437d32ceefaec7161803230c7a20bc1b4d91"
   integrity sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig==
@@ -529,7 +534,15 @@ dom-serializer@^1.0.1, dom-serializer@^1.3.2:
     domhandler "^4.2.0"
     entities "^2.0.0"
 
-domelementtype@1:
+dom-serializer@~0.1.0:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
+  integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==
+  dependencies:
+    domelementtype "^1.3.0"
+    entities "^1.1.1"
+
+domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f"
   integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==
@@ -546,7 +559,14 @@ domhandler@2.0:
   dependencies:
     domelementtype "1"
 
-domhandler@^4.0.0, domhandler@^4.2.0, domhandler@^4.2.2:
+domhandler@^2.3.0:
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803"
+  integrity sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==
+  dependencies:
+    domelementtype "1"
+
+domhandler@^4.2.0, domhandler@^4.2.2:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.2.2.tgz#e825d721d19a86b8c201a35264e226c678ee755f"
   integrity sha512-PzE9aBMsdZO8TK4BnuJwH0QT41wgMbRzuZrHUcpYncEjmQazq8QEaBWgLG7ZyC/DAZKEgglpIA6j4Qn/HmxS3w==
@@ -560,7 +580,23 @@ domutils@1.1:
   dependencies:
     domelementtype "1"
 
-domutils@^2.5.2, domutils@^2.6.0, domutils@^2.7.0, domutils@^2.8.0:
+domutils@1.5.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf"
+  integrity sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=
+  dependencies:
+    dom-serializer "0"
+    domelementtype "1"
+
+domutils@^1.5.1:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a"
+  integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==
+  dependencies:
+    dom-serializer "0"
+    domelementtype "1"
+
+domutils@^2.8.0:
   version "2.8.0"
   resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135"
   integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==
@@ -609,6 +645,11 @@ entities@1.1.1:
   resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0"
   integrity sha1-blwtClYhtdra7O+AuQ7ftc13cvA=
 
+entities@^1.1.1, entities@~1.1.1:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
+  integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
+
 entities@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55"
@@ -894,15 +935,17 @@ htmlparser2@3.0:
     domutils "1.1"
     readable-stream "1.0"
 
-htmlparser2@^6.1.0:
-  version "6.1.0"
-  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-6.1.0.tgz#c4d762b6c3371a05dbe65e94ae43a9f845fb8fb7"
-  integrity sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==
+htmlparser2@^3.9.1:
+  version "3.10.1"
+  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f"
+  integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==
   dependencies:
-    domelementtype "^2.0.1"
-    domhandler "^4.0.0"
-    domutils "^2.5.2"
-    entities "^2.0.0"
+    domelementtype "^1.3.1"
+    domhandler "^2.3.0"
+    domutils "^1.5.1"
+    entities "^1.1.1"
+    inherits "^2.0.1"
+    readable-stream "^3.1.1"
 
 htmlparser2@^7.1.2:
   version "7.1.2"
@@ -990,7 +1033,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2, inherits@2.0.4, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3:
+inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.1, inherits@~2.0.3:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -1121,6 +1164,36 @@ lodash.assign@^4.2.0:
   resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
   integrity sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=
 
+lodash.assignin@^4.0.9:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.assignin/-/lodash.assignin-4.2.0.tgz#ba8df5fb841eb0a3e8044232b0e263a8dc6a28a2"
+  integrity sha1-uo31+4QesKPoBEIysOJjqNxqKKI=
+
+lodash.bind@^4.1.4:
+  version "4.2.1"
+  resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35"
+  integrity sha1-euMBfpOWIqwxt9fX3LGzTbFpDTU=
+
+lodash.defaults@^4.0.1:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
+
+lodash.filter@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
+  integrity sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4=
+
+lodash.flatten@^4.2.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
+  integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
+
+lodash.foreach@^4.3.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53"
+  integrity sha1-Gmo16s5AEoDH8G3d7DUWWrJ+PlM=
+
 lodash.get@^4.4.2:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
@@ -1131,6 +1204,36 @@ lodash.has@^4.5.2:
   resolved "https://registry.yarnpkg.com/lodash.has/-/lodash.has-4.5.2.tgz#d19f4dc1095058cccbe2b0cdf4ee0fe4aa37c862"
   integrity sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=
 
+lodash.map@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.map/-/lodash.map-4.6.0.tgz#771ec7839e3473d9c4cde28b19394c3562f4f6d3"
+  integrity sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=
+
+lodash.merge@^4.4.0:
+  version "4.6.2"
+  resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
+  integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==
+
+lodash.pick@^4.2.1:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
+  integrity sha1-UvBWEP/53tQiYRRB7R/BI6AwAbM=
+
+lodash.reduce@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.reduce/-/lodash.reduce-4.6.0.tgz#f1ab6b839299ad48f784abbf476596f03b914d3b"
+  integrity sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs=
+
+lodash.reject@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.reject/-/lodash.reject-4.6.0.tgz#80d6492dc1470864bbf583533b651f42a9f52415"
+  integrity sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU=
+
+lodash.some@^4.4.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d"
+  integrity sha1-G7nzFO9ri63tE7VJFpsqlF62jk0=
+
 lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
@@ -1380,12 +1483,12 @@ normalize-path@^3.0.0, normalize-path@~3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
-nth-check@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.0.1.tgz#2efe162f5c3da06a28959fbd3db75dbeea9f0fc2"
-  integrity sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==
+nth-check@~1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c"
+  integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==
   dependencies:
-    boolbase "^1.0.0"
+    boolbase "~1.0.0"
 
 oauth-sign@~0.9.0:
   version "0.9.0"
@@ -1441,18 +1544,6 @@ parse-iso-duration@1.0.0:
   resolved "https://registry.yarnpkg.com/parse-iso-duration/-/parse-iso-duration-1.0.0.tgz#b923ab898a8ff8f42bdc9ee5db6e22808c48a864"
   integrity sha1-uSOriYqP+PQr3J7l224igIxIqGQ=
 
-parse5-htmlparser2-tree-adapter@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz#2cdf9ad823321140370d4dbf5d3e92c7c8ddc6e6"
-  integrity sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==
-  dependencies:
-    parse5 "^6.0.1"
-
-parse5@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b"
-  integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==
-
 parseurl@~1.3.3:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
@@ -1590,6 +1681,15 @@ readable-stream@^2.2.2, readable-stream@^2.3.5:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
+readable-stream@^3.1.1:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
+  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
 readable-stream@~2.1.5:
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.1.5.tgz#66fa8b720e1438b364681f2ad1a63c618448c9d0"
@@ -1723,7 +1823,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
-safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2:
+safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
@@ -1872,6 +1972,13 @@ stream-parser@~0.3.1:
   dependencies:
     debug "2"
 
+string_decoder@^1.1.1:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
+  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
+  dependencies:
+    safe-buffer "~5.2.0"
+
 string_decoder@~0.10.x:
   version "0.10.31"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
@@ -1942,11 +2049,6 @@ tough-cookie@~2.5.0:
     psl "^1.1.28"
     punycode "^2.1.1"
 
-tslib@^2.2.0:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
-  integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==
-
 tunnel-agent@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
@@ -2001,7 +2103,7 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
-util-deprecate@~1.0.1:
+util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
   integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=

From 063c21e367c7652c76a017dda7b67be98c502160 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 28 Oct 2021 20:33:37 +0300
Subject: [PATCH 076/102] fix cheerio import

---
 lib/html-utils.js                             | 20 +++++++++----------
 lib/plugins/system/cheerio.js                 |  3 +--
 lib/plugins/system/oembed/oembed.js           |  4 ++--
 modules/tests-ui/utils.js                     |  6 +++---
 .../domains/instagram.com/instagram.com.js    |  4 ++--
 plugins/domains/tumblr.com/tumblr.api.js      |  8 ++++----
 plugins/domains/tumblr.com/tumblr.photo.js    |  6 +++---
 plugins/domains/tumblr.com/tumblr.text.js     |  6 +++---
 plugins/domains/tumblr.com/tumblr.video.js    |  6 +++---
 plugins/domains/youtube.com/youtube.video.js  |  4 ++--
 plugins/links/embedURL/ld-video.js            |  4 ++--
 plugins/links/oembed-video.js                 |  4 ++--
 plugins/meta/ld-article.js                    |  4 ++--
 plugins/meta/ld-product.js                    |  4 ++--
 plugins/meta/oembed-description.js            |  4 ++--
 15 files changed, 43 insertions(+), 44 deletions(-)

diff --git a/lib/html-utils.js b/lib/html-utils.js
index 00d93132c..e8a44c0e9 100644
--- a/lib/html-utils.js
+++ b/lib/html-utils.js
@@ -1,5 +1,5 @@
-    import cheerio_pkg from 'cheerio';
-    const $ = cheerio_pkg.default;
+    import cheerio from 'cheerio';
+    
     import * as _ from 'underscore';
     import CONFIG from '../config.js';
 
@@ -30,7 +30,7 @@
             }
         }
 
-        var $container = $('<div>')
+        var $container = cheerio('<div>')
             .append($element);
 
         if (aspectWrapperClass) {
@@ -45,7 +45,7 @@
         var hasMaxWidth = media && (media["max-width"] || media["min-width"] || media["width"] || verticalAspect);
 
         if (hasMaxWidth || forceWidthLimitContainer) {
-            $widthLimitContainer = $('<div>')
+            $widthLimitContainer = cheerio('<div>')
                 .append($container);
         }
 
@@ -166,7 +166,7 @@
                     && data.href;
             },
             generate: function(data) {
-                var $img = $('<img>')
+                var $img = cheerio('<img>')
                     .attr('src', data.href);
                 if (data.title) {
                     $img
@@ -190,7 +190,7 @@
                 var givf = data.rel.indexOf('gifv') > -1;
                 var autoplay = data.rel.indexOf('autoplay') > -1 || givf;
 
-                var $video = $('<video' + (givf ? ' loop muted webkit-playsinline' : ' controls') + (autoplay ? ' autoplay' : '') + '>Your browser does not support HTML5 video.</video>');
+                var $video = cheerio('<video' + (givf ? ' loop muted webkit-playsinline' : ' controls') + (autoplay ? ' autoplay' : '') + '>Your browser does not support HTML5 video.</video>');
 
                 if (iframelyData && iframelyData.links) {
 
@@ -280,7 +280,7 @@
             },
             generate: function(data, options) {
 
-                var $iframe = $('<iframe>')
+                var $iframe = cheerio('<iframe>')
                     .attr('src', data.href)
                     .css('border', '0')
                     .attr('allowfullscreen', '');
@@ -333,11 +333,11 @@
     export function generateElementWrapperHtml(element, link, options) {
 
         if (typeof element === 'string') {
-            element = $(element);
+            element = cheerio(element);
         }
 
         var $el = wrapContainer(element, link, options);
-        return $('<div>').append($el).html();
+        return cheerio('<div>').append($el).html();
     };
 
     export function generateLinkElementHtml(link, options) {
@@ -348,7 +348,7 @@
             if (options && options.canonical && link.href !== options.canonical) {
                 $el.attr('data-embed-canonical', options.canonical);
             }
-            return $('<div>').append($el).html();
+            return cheerio('<div>').append($el).html();
         } else {
             return '';
         }
diff --git a/lib/plugins/system/cheerio.js b/lib/plugins/system/cheerio.js
index 9973f2ea4..d5832287e 100644
--- a/lib/plugins/system/cheerio.js
+++ b/lib/plugins/system/cheerio.js
@@ -1,5 +1,4 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
 import htmlparser2 from "htmlparser2";
 var DomHandler = htmlparser2.DomHandler;
 
diff --git a/lib/plugins/system/oembed/oembed.js b/lib/plugins/system/oembed/oembed.js
index a6b2213a8..1d2bebba7 100644
--- a/lib/plugins/system/oembed/oembed.js
+++ b/lib/plugins/system/oembed/oembed.js
@@ -1,6 +1,6 @@
 import * as oembedUtils from './oembedUtils.js';
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 import * as entities from 'entities';
 import * as URL from 'url';
 import * as querystring from 'querystring';
diff --git a/modules/tests-ui/utils.js b/modules/tests-ui/utils.js
index de29ce6ab..6d0f58308 100644
--- a/modules/tests-ui/utils.js
+++ b/modules/tests-ui/utils.js
@@ -355,14 +355,14 @@ export function fetchUrlsByPageAndSelector(page, selector, options, cb) {
             iframelyGetPluginData(page, 'cheerio', findWhitelistRecordFor, cb);
         },
 
-        function($, cb) {
+        function(cheerio, cb) {
 
-            var $links = $(selector);
+            var $links = cheerio(selector);
 
             var urls = [];
             $links.each(function() {
                 if (urls.length < MAX_FEED_URLS) {
-                    var href = $(this).attr(options.urlAttribute || "href");
+                    var href = cheerio(this).attr(options.urlAttribute || "href");
                     if (href) {
                         var href = url.resolve(page, href);
                         if (urls.indexOf(href) == -1) {
diff --git a/plugins/domains/instagram.com/instagram.com.js b/plugins/domains/instagram.com/instagram.com.js
index 7276ac95e..63eeb4849 100644
--- a/plugins/domains/instagram.com/instagram.com.js
+++ b/plugins/domains/instagram.com/instagram.com.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 import { decodeHTML5 } from 'entities';
 
 export default {
diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js
index f696f985a..33f45a02f 100644
--- a/plugins/domains/tumblr.com/tumblr.api.js
+++ b/plugins/domains/tumblr.com/tumblr.api.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 
 export default {
 
@@ -12,7 +12,7 @@ export default {
 
     getMeta: function(tumblr_post) {
 
-        var caption = tumblr_post.caption ? $('<div>').html(tumblr_post.caption).text() : "";
+        var caption = tumblr_post.caption ? cheerio('<div>').html(tumblr_post.caption).text() : "";
         if (caption && caption.length > 160) {
             caption = caption.split(/[.,!?]/)[0];
         }
@@ -27,7 +27,7 @@ export default {
             shortlink: tumblr_post.short_url,
             date: tumblr_post.date,
             duration: tumblr_post.duration,
-            description: tumblr_post.body && /<p/.test(tumblr_post.body) ? $('<div>').html(tumblr_post.body).find('p').first().text() : null
+            description: tumblr_post.body && /<p/.test(tumblr_post.body) ? cheerio('<div>').html(tumblr_post.body).find('p').first().text() : null
         };
     },
 
diff --git a/plugins/domains/tumblr.com/tumblr.photo.js b/plugins/domains/tumblr.com/tumblr.photo.js
index 1b73ad6fc..914f6ec7b 100644
--- a/plugins/domains/tumblr.com/tumblr.photo.js
+++ b/plugins/domains/tumblr.com/tumblr.photo.js
@@ -1,6 +1,6 @@
 import * as _ from 'underscore';
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 
 import tumblr_api from './tumblr.api.js';
 
@@ -30,7 +30,7 @@ export default {
         tumblr_post.photos.forEach(function(photo) {
 
             var title = photo.caption || tumblr_post.caption;
-            title = $('<div>').html(title).text();
+            title = cheerio('<div>').html(title).text();
             if (title && title.length > 160) {
                 title = title.split(/[.,!?]/)[0];
             }
diff --git a/plugins/domains/tumblr.com/tumblr.text.js b/plugins/domains/tumblr.com/tumblr.text.js
index b857edbae..dc3d59222 100644
--- a/plugins/domains/tumblr.com/tumblr.text.js
+++ b/plugins/domains/tumblr.com/tumblr.text.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 import tumblr_api from './tumblr.api.js';
 
 export default {
@@ -20,7 +20,7 @@ export default {
             return;
         }
 
-        var $post = $('<div>').html(tumblr_post.body);
+        var $post = cheerio('<div>').html(tumblr_post.body);
         var $image = $post.find('img').first(); // Could be more than 1 image, true. But the response time will be unacceptable as post-processing will check all image sizes.
 
         if ($image ) {
diff --git a/plugins/domains/tumblr.com/tumblr.video.js b/plugins/domains/tumblr.com/tumblr.video.js
index 2a864ff58..7a3f02ff6 100644
--- a/plugins/domains/tumblr.com/tumblr.video.js
+++ b/plugins/domains/tumblr.com/tumblr.video.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const $ = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 import tumblr_api from './tumblr.api.js';
 
 export default {
@@ -26,7 +26,7 @@ export default {
 
             var p = tumblr_post.player instanceof Array ? tumblr_post.player[0] : tumblr_post.player;
 
-            var $c = $('<div>').append(p.embed_code);
+            var $c = cheerio('<div>').append(p.embed_code);
             var $iframe = $c.find('iframe');
 
             if ($iframe.length) {
diff --git a/plugins/domains/youtube.com/youtube.video.js b/plugins/domains/youtube.com/youtube.video.js
index a56a353a6..26d8d262e 100644
--- a/plugins/domains/youtube.com/youtube.video.js
+++ b/plugins/domains/youtube.com/youtube.video.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 import * as querystring from 'querystring';
 import * as _ from 'underscore';
 import log from '../../../logging.js'
diff --git a/plugins/links/embedURL/ld-video.js b/plugins/links/embedURL/ld-video.js
index 847185a2f..cc0e978a7 100644
--- a/plugins/links/embedURL/ld-video.js
+++ b/plugins/links/embedURL/ld-video.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 
 export default {
 
diff --git a/plugins/links/oembed-video.js b/plugins/links/oembed-video.js
index 3c22ac507..aed329564 100644
--- a/plugins/links/oembed-video.js
+++ b/plugins/links/oembed-video.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 import * as entities from 'entities';
 
 export default {
diff --git a/plugins/meta/ld-article.js b/plugins/meta/ld-article.js
index 228c906dd..068e273b5 100644
--- a/plugins/meta/ld-article.js
+++ b/plugins/meta/ld-article.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 
 export default {
 
diff --git a/plugins/meta/ld-product.js b/plugins/meta/ld-product.js
index bfb07cc77..c2e7df3b9 100644
--- a/plugins/meta/ld-product.js
+++ b/plugins/meta/ld-product.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 
 export default {
 
diff --git a/plugins/meta/oembed-description.js b/plugins/meta/oembed-description.js
index 021f0d040..d3543b88b 100644
--- a/plugins/meta/oembed-description.js
+++ b/plugins/meta/oembed-description.js
@@ -1,5 +1,5 @@
-import cheerio_pkg from 'cheerio';
-const cheerio = cheerio_pkg.default;
+import cheerio from 'cheerio';
+
 
 export default {
 

From afa68a3cd997554fea69bd8df9386fa5b931a6b7 Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Thu, 28 Oct 2021 15:11:44 -0400
Subject: [PATCH 077/102] remove oembed-iframe

---
 plugins/custom/oembed-iframe.js | 13 -------------
 plugins/domains/libsyn.com.js   |  6 +-----
 plugins/domains/spotify.com.js  |  2 +-
 3 files changed, 2 insertions(+), 19 deletions(-)
 delete mode 100644 plugins/custom/oembed-iframe.js

diff --git a/plugins/custom/oembed-iframe.js b/plugins/custom/oembed-iframe.js
deleted file mode 100644
index 2e997eec5..000000000
--- a/plugins/custom/oembed-iframe.js
+++ /dev/null
@@ -1,13 +0,0 @@
-export default {
-
-    provides: "iframe",
-
-    getData: function(oembed) {
-
-        if (oembed.getIframe()) {
-            return {
-                iframe: oembed.getIframe()
-            }
-        }
-    }
-};
\ No newline at end of file
diff --git a/plugins/domains/libsyn.com.js b/plugins/domains/libsyn.com.js
index 7e4617e00..ded2ab83a 100644
--- a/plugins/domains/libsyn.com.js
+++ b/plugins/domains/libsyn.com.js
@@ -2,10 +2,7 @@ export default {
 
     re: /^https?:\/\/[a-zA-Z0-9\.]+\.libsyn(?:pro)?\.com\//,
 
-    mixins: [
-        "*",
-        "oembed-iframe"
-    ],
+    mixins: ["*"],
 
     getLink: function(iframe, meta, options) {
 
@@ -91,7 +88,6 @@ export default {
     },
 
     tests: [
-        "https://3manbreak.libsyn.com/10-build-a-bear-for-bradley-beal-december-1-of-3",
         "https://directory.libsyn.com/episode/index/id/3252958",
         // "http://mohrstories.libsyn.com/podcast/mor-stories-267-john-dimaggio", // not supported
         "https://lowcarbmd.libsyn.com/episode-20-in-defense-of-docs",
diff --git a/plugins/domains/spotify.com.js b/plugins/domains/spotify.com.js
index 08414f13e..25801fd39 100644
--- a/plugins/domains/spotify.com.js
+++ b/plugins/domains/spotify.com.js
@@ -97,7 +97,7 @@ export default {
         }
     },    
 
-    tests: [{noFeeds: true}, {skipMethods: ["getData"], skipMixins: ["oembed-iframe", "oembed-thumbnail", "og-image"]},
+    tests: [{noFeeds: true}, {skipMethods: ["getData"], skipMixins: ["oembed-thumbnail", "og-image"]},
         "https://open.spotify.com/playlist/44CgBWWr6nlpy7bdZS8ZmN",
         "http://open.spotify.com/track/6ol4ZSifr7r3Lb2a9L5ZAB",
         "https://open.spotify.com/playlist/4SsKyjaGlrHJbRCQwpeUsz",

From a6175a751671a6825b0f24aeda87725363f50bd4 Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Thu, 28 Oct 2021 16:07:17 -0400
Subject: [PATCH 078/102] Revert "remove oembed-iframe"

This reverts commit afa68a3cd997554fea69bd8df9386fa5b931a6b7.
---
 plugins/custom/oembed-iframe.js | 13 +++++++++++++
 plugins/domains/libsyn.com.js   |  6 +++++-
 plugins/domains/spotify.com.js  |  2 +-
 3 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 plugins/custom/oembed-iframe.js

diff --git a/plugins/custom/oembed-iframe.js b/plugins/custom/oembed-iframe.js
new file mode 100644
index 000000000..2e997eec5
--- /dev/null
+++ b/plugins/custom/oembed-iframe.js
@@ -0,0 +1,13 @@
+export default {
+
+    provides: "iframe",
+
+    getData: function(oembed) {
+
+        if (oembed.getIframe()) {
+            return {
+                iframe: oembed.getIframe()
+            }
+        }
+    }
+};
\ No newline at end of file
diff --git a/plugins/domains/libsyn.com.js b/plugins/domains/libsyn.com.js
index ded2ab83a..7e4617e00 100644
--- a/plugins/domains/libsyn.com.js
+++ b/plugins/domains/libsyn.com.js
@@ -2,7 +2,10 @@ export default {
 
     re: /^https?:\/\/[a-zA-Z0-9\.]+\.libsyn(?:pro)?\.com\//,
 
-    mixins: ["*"],
+    mixins: [
+        "*",
+        "oembed-iframe"
+    ],
 
     getLink: function(iframe, meta, options) {
 
@@ -88,6 +91,7 @@ export default {
     },
 
     tests: [
+        "https://3manbreak.libsyn.com/10-build-a-bear-for-bradley-beal-december-1-of-3",
         "https://directory.libsyn.com/episode/index/id/3252958",
         // "http://mohrstories.libsyn.com/podcast/mor-stories-267-john-dimaggio", // not supported
         "https://lowcarbmd.libsyn.com/episode-20-in-defense-of-docs",
diff --git a/plugins/domains/spotify.com.js b/plugins/domains/spotify.com.js
index 25801fd39..08414f13e 100644
--- a/plugins/domains/spotify.com.js
+++ b/plugins/domains/spotify.com.js
@@ -97,7 +97,7 @@ export default {
         }
     },    
 
-    tests: [{noFeeds: true}, {skipMethods: ["getData"], skipMixins: ["oembed-thumbnail", "og-image"]},
+    tests: [{noFeeds: true}, {skipMethods: ["getData"], skipMixins: ["oembed-iframe", "oembed-thumbnail", "og-image"]},
         "https://open.spotify.com/playlist/44CgBWWr6nlpy7bdZS8ZmN",
         "http://open.spotify.com/track/6ol4ZSifr7r3Lb2a9L5ZAB",
         "https://open.spotify.com/playlist/4SsKyjaGlrHJbRCQwpeUsz",

From a35e7549796a779761a616e4d7b6d1625b2d8015 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 28 Oct 2021 23:09:04 +0300
Subject: [PATCH 079/102] test skipping plugins for mandatory params

---
 lib/core.js | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index 7ef9421d6..99c71cdb7 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -347,7 +347,7 @@
 
         var loadedParams = Object.keys(context);
 
-        var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams));
+        // var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams));
 
         // Reset scanned plugins for each iteration.
         var scannedPluginsIds = {};
@@ -362,16 +362,16 @@
         }
 
         // If has new unused params (mandatoryParams) - then find plugins which can use them.
-        if (mandatoryParams && mandatoryParams.length > 0) {
+        // if (mandatoryParams && mandatoryParams.length > 0) {
 
-            var secondaryPlugins = findPluginsForMandatoryParams(mandatoryParams, usedDomains);
+        //     var secondaryPlugins = findPluginsForMandatoryParams(mandatoryParams, usedDomains);
 
-            // Find methods in plugins, which can use mandatory params.
-            for(var i = 0; i < secondaryPlugins.length; i++) {
-                var pluginId = secondaryPlugins[i];
-                findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams);
-            }
-        }
+        //     // Find methods in plugins, which can use mandatory params.
+        //     for(var i = 0; i < secondaryPlugins.length; i++) {
+        //         var pluginId = secondaryPlugins[i];
+        //         findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams);
+        //     }
+        // }
 
         // Run found methods.
         runMethods(methods, context, pluginsUrlMatches, options, asyncMethodCb);

From 4d309e5b1e9e8d3c77578fddf32b3eabe92df52c Mon Sep 17 00:00:00 2001
From: Ivan Paramonau <i.paramonau@gmail.com>
Date: Thu, 28 Oct 2021 16:43:20 -0400
Subject: [PATCH 080/102] review oembed-iframe

---
 lib/plugins/system/oembed/oembed.js                 |  6 +-----
 plugins/domains/art19.com.js                        |  3 ++-
 .../brightcove.com/players.brightcove.net.js        |  3 ++-
 plugins/domains/cartodb.com.js                      |  3 ++-
 plugins/domains/cnevids.com.js                      | 13 +++++--------
 plugins/domains/codepen.io.js                       |  1 +
 plugins/domains/dailymotion.com/dailymotion.com.js  | 11 +++++------
 plugins/domains/libsyn.com.js                       |  3 +--
 plugins/domains/mixcloud.com.js                     |  1 +
 plugins/domains/slideshare.net.js                   |  3 ++-
 plugins/domains/soundcloud.com/soundcloud.com.js    |  1 +
 plugins/domains/speakerdeck.com.js                  |  2 --
 plugins/domains/spotify.com.js                      |  1 +
 plugins/{custom => links}/oembed-iframe.js          |  0
 14 files changed, 24 insertions(+), 27 deletions(-)
 rename plugins/{custom => links}/oembed-iframe.js (100%)

diff --git a/lib/plugins/system/oembed/oembed.js b/lib/plugins/system/oembed/oembed.js
index 1d2bebba7..63369052b 100644
--- a/lib/plugins/system/oembed/oembed.js
+++ b/lib/plugins/system/oembed/oembed.js
@@ -91,7 +91,7 @@ function getOembedIframeAttr(oembed) {
 
 export default {
 
-    provides: ['self', 'oembedError', 'iframe'],
+    provides: ['self', 'oembedError'],
 
     getData: function(url, oembedLinks, options, cb) {
 
@@ -139,10 +139,6 @@ export default {
                 }, oembed),
             };
 
-            if (oembed.html && _getOembedIframe(oembed)) {
-                result.iframe = _getOembedIframe(oembed)
-            } 
-
             // If no oEmbed record for the domain - allow to be whitelisted by the oEmbed endpoint domian record.
             if (options.getWhitelistRecord) {
                 var currentWhitelistRecord = options.getWhitelistRecord(url, {disableWildcard: true});
diff --git a/plugins/domains/art19.com.js b/plugins/domains/art19.com.js
index 7ca36544c..fdf55b2dd 100644
--- a/plugins/domains/art19.com.js
+++ b/plugins/domains/art19.com.js
@@ -10,7 +10,8 @@ export default {
         "oembed-description",
         "og-image",
         "oembed-site",
-        "domain-icon"
+        "domain-icon",
+        "oembed-iframe"
     ],
 
     getLink: function(iframe, options) {
diff --git a/plugins/domains/brightcove.com/players.brightcove.net.js b/plugins/domains/brightcove.com/players.brightcove.net.js
index 90a74d64b..c0e1d164f 100644
--- a/plugins/domains/brightcove.com/players.brightcove.net.js
+++ b/plugins/domains/brightcove.com/players.brightcove.net.js
@@ -11,7 +11,8 @@ export default {
     mixins: [
         "oembed-title",
         "oembed-site",
-        "oembed-error"
+        "oembed-error",
+        "oembed-iframe"
     ],
 
     //HTML parser will 404 if BC account or player does not exist.
diff --git a/plugins/domains/cartodb.com.js b/plugins/domains/cartodb.com.js
index 129f7236b..8ebbf26ab 100644
--- a/plugins/domains/cartodb.com.js
+++ b/plugins/domains/cartodb.com.js
@@ -11,7 +11,8 @@ export default {
         "oembed-author",
         "oembed-site",
         "keywords",
-        "favicon"
+        "favicon",
+        "oembed-iframe"
     ],
 
     getMeta: function(url, meta) {
diff --git a/plugins/domains/cnevids.com.js b/plugins/domains/cnevids.com.js
index cc7773bfe..105d6c72e 100644
--- a/plugins/domains/cnevids.com.js
+++ b/plugins/domains/cnevids.com.js
@@ -4,19 +4,16 @@ export default {
         "oembed-thumbnail",
         "oembed-author",
         "oembed-site",
-        "oembed-title"
+        "oembed-title",
+        "oembed-iframe"
     ],
 
-    getLink: function(oembed, whitelistRecord) {
+    getLink: function(iframe, oembed, whitelistRecord) {
 
-        var iframe = oembed.getIframe();
-
-        if (iframe && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.video', 'autoplay')) {
-
-            var href = iframe.src;
+        if (iframe.src && whitelistRecord.isAllowed && whitelistRecord.isAllowed('oembed.video', 'autoplay')) {
 
             var links = [{
-                href: href + (href.indexOf('?') > -1 ? '&' : '?') + 'autoplay=0',
+                href: iframe.replaceQuerystring({autoplay:0}),
                 type: CONFIG.T.text_html,
                 rel: [CONFIG.R.player, CONFIG.R.oembed, CONFIG.R.html5],
                 autoplay: "autoplay=1",
diff --git a/plugins/domains/codepen.io.js b/plugins/domains/codepen.io.js
index 006b02b62..05dc128a3 100644
--- a/plugins/domains/codepen.io.js
+++ b/plugins/domains/codepen.io.js
@@ -7,6 +7,7 @@ export default {
         "oembed-author",
         "oembed-site",
         "oembed-title",
+        "oembed-iframe",
         //"description", // don't enable to avoid 403 from CodePen's htmlparser. Description is '...' in most cases anyway
         "domain-icon"
     ],
diff --git a/plugins/domains/dailymotion.com/dailymotion.com.js b/plugins/domains/dailymotion.com/dailymotion.com.js
index e94590427..1940d03c6 100644
--- a/plugins/domains/dailymotion.com/dailymotion.com.js
+++ b/plugins/domains/dailymotion.com/dailymotion.com.js
@@ -11,6 +11,7 @@ export default {
         "domain-icon",
         "og-description",
         "canonical",
+        "oembed-iframe",
         "video"
     ],
 
@@ -19,17 +20,15 @@ export default {
      *   - queue-enable=false  - https://faq.dailymotion.com/hc/en-us/articles/360000713928-Disabling-the-Up-Next-Queue
      *   - ui-start-screen-info=0 - hide title amontg other things - https://nextgenthemes.com/how-to-hide-titles-and-change-other-setting-for-youtube-vimeo-embeds-in-wordpress-with-arve/
      */
-    getLink: function (url, oembed, options) {
+    getLink: function (url, iframe, options) {
         var playlistParams = querystring.parse(options.getProviderOptions('dailymotion.get_params', '').replace(/^\?/, ''));
-        var qs = querystring.stringify(playlistParams);
-        var href = oembed.getIframeAttr('src');
 
-        if (href && oembed.height) {
+        if (iframe.src && iframe.height) {
             return {
-                href: href + (href.indexOf("?") > -1 ? "&" : (qs !== "" ? "?" : "")) + qs,
+                href: iframe.replaceQuerystring(playlistParams),
                 type: CONFIG.T.text_html,
                 "rel": [CONFIG.R.player, CONFIG.R.html5, CONFIG.R.ssl, CONFIG.R.oembed],
-                "aspect-ratio": oembed.width / oembed.height,
+                "aspect-ratio": iframe.width / iframe.height,
                 scrolling: 'no',
                 autoplay: "autoplay=1"
             };
diff --git a/plugins/domains/libsyn.com.js b/plugins/domains/libsyn.com.js
index 7e4617e00..67c9274ae 100644
--- a/plugins/domains/libsyn.com.js
+++ b/plugins/domains/libsyn.com.js
@@ -3,8 +3,7 @@ export default {
     re: /^https?:\/\/[a-zA-Z0-9\.]+\.libsyn(?:pro)?\.com\//,
 
     mixins: [
-        "*",
-        "oembed-iframe"
+        "*"
     ],
 
     getLink: function(iframe, meta, options) {
diff --git a/plugins/domains/mixcloud.com.js b/plugins/domains/mixcloud.com.js
index 9e3a918e6..67fb909ba 100644
--- a/plugins/domains/mixcloud.com.js
+++ b/plugins/domains/mixcloud.com.js
@@ -11,6 +11,7 @@ export default {
         "oembed-author",
         "oembed-site",
         "oembed-error",
+        "oembed-iframe",
         "domain-icon"
     ],
 
diff --git a/plugins/domains/slideshare.net.js b/plugins/domains/slideshare.net.js
index 42a686ecd..5ac94e6c9 100644
--- a/plugins/domains/slideshare.net.js
+++ b/plugins/domains/slideshare.net.js
@@ -11,7 +11,8 @@ export default {
             "canonical",
             "description",
             "oembed-site",
-            "oembed-title"
+            "oembed-title",
+            "oembed-iframe"
     ],
 
     getMeta: function(meta) {
diff --git a/plugins/domains/soundcloud.com/soundcloud.com.js b/plugins/domains/soundcloud.com/soundcloud.com.js
index 905a2365a..a27734e83 100644
--- a/plugins/domains/soundcloud.com/soundcloud.com.js
+++ b/plugins/domains/soundcloud.com/soundcloud.com.js
@@ -7,6 +7,7 @@ export default {
         "oembed-site",
         "oembed-author",
         "oembed-description",
+        "oembed-iframe",
         // do not link to meta as it disables support for direct player urls redirects from w.soundcloud.com
         "domain-icon"
     ],
diff --git a/plugins/domains/speakerdeck.com.js b/plugins/domains/speakerdeck.com.js
index 587ba1143..c732438b1 100644
--- a/plugins/domains/speakerdeck.com.js
+++ b/plugins/domains/speakerdeck.com.js
@@ -8,8 +8,6 @@ export default {
 
         var slide = options.getRequestOptions('speakerdeck.slide', query.slide ? parseInt(query.slide) || 1 : 1);
 
-        console.log('slide', slide);
-
         if (iframe.src && iframe.width && iframe.height) {
             var href = iframe.src.replace(/^\/\//, 'https://');
             if (slide > 1) {
diff --git a/plugins/domains/spotify.com.js b/plugins/domains/spotify.com.js
index 08414f13e..2b7adf0bf 100644
--- a/plugins/domains/spotify.com.js
+++ b/plugins/domains/spotify.com.js
@@ -8,6 +8,7 @@ export default {
         "oembed-title",
         "og-image",
         "oembed-thumbnail",
+        "oembed-iframe",
         "domain-icon"
     ],
 
diff --git a/plugins/custom/oembed-iframe.js b/plugins/links/oembed-iframe.js
similarity index 100%
rename from plugins/custom/oembed-iframe.js
rename to plugins/links/oembed-iframe.js

From 488bec7282ea463bba9418915480e911a3fe2e62 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 29 Oct 2021 18:19:37 +0300
Subject: [PATCH 081/102] temp fix AbortError for uri status

---
 lib/utils.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/utils.js b/lib/utils.js
index 1ea987944..333c59bcd 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1015,7 +1015,8 @@ var getUriStatusPrivate = function(uri, options, cb) {
     try {
         fetchStream(request_options)
             .then(res => {
-                res.abortController.abort();
+                // TODO: may cause AbortError before cb.
+                //res.abortController.abort();
                 var data = {
                     code: res.status,
                     content_type: res.headers && res.headers['content-type'],

From 4cf4f7493811f1c2ec48ed32d1a1696b2995cf05 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 1 Nov 2021 20:39:40 +0200
Subject: [PATCH 082/102] fix `__dirname` in `utils.js`

---
 utils.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/utils.js b/utils.js
index 50b9d6160..cd1f2e90b 100644
--- a/utils.js
+++ b/utils.js
@@ -14,6 +14,12 @@
     import * as whitelist from './lib/whitelist.js';
     import * as pluginLoader from './lib/loader/pluginLoader.js';
 
+    import { fileURLToPath } from 'url';
+    import { dirname } from 'path';
+
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+
     export function NotFound(message, messages) {
 
         if (typeof message === 'object') {

From a495cee07cd821b4dd1274b984c95d5d26335edd Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 1 Nov 2021 20:39:57 +0200
Subject: [PATCH 083/102] fix `package.json` `main`

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 554395e6d..63742637d 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,7 @@
         "supertest": "^4.0.2"
     },
     "iframely-proxy-plugins": true,
-    "main": "./lib/core",
+    "main": "./lib/core.js",
     "scripts": {
         "test": "npm run test-core-plugins && npm run test-e2e",
         "test-core-plugins": "mocha --exit test/core-plugins.js",

From 726413454b954f7326629998c62a280634b37a27 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 5 Nov 2021 22:53:05 +0200
Subject: [PATCH 084/102] fix import `GracefulServer`

---
 server.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server.js b/server.js
index 48d52b107..6e969f166 100644
--- a/server.js
+++ b/server.js
@@ -16,8 +16,9 @@ console.log(' - support@iframely.com - if you need help');
 console.log(' - twitter.com/iframely - news & updates');
 console.log(' - github.com/itteco/iframely - star & contribute');
 
+import { GracefulServer } from 'graceful-cluster';
+
 if (!CONFIG.DEBUG) {
-    var GracefulServer = require('graceful-cluster').GracefulServer;
     new GracefulServer({
         server: server,
         log: sysUtils.log,

From a7da5b10566509d28e19300d1bf3d0894c6f5d90 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 5 Nov 2021 22:53:56 +0200
Subject: [PATCH 085/102] fix `tester.js` for `pm2`

---
 tester.js | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tester.js

diff --git a/tester.js b/tester.js
new file mode 100644
index 000000000..8d6bfef0e
--- /dev/null
+++ b/tester.js
@@ -0,0 +1 @@
+import './modules/tests-ui/tester.js';
\ No newline at end of file

From f79e650200cf09000e8617f81ff033a89d8fe911 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 8 Nov 2021 23:24:53 +0200
Subject: [PATCH 086/102] disable decode html entities in htmlparser2

---
 lib/plugins/system/htmlparser/htmlparser.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 1789f6ab2..175b65b4f 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -140,7 +140,8 @@ export default {
                     }
                 };
                 var parser = new Parser(handler, {
-                    lowerCaseTags: true
+                    lowerCaseTags: true,
+                    decodeEntities: false   // Fixes decoding html characters like &nbsp; to UTF-8, we have own decoders.
                 });
                 handler.headers = headers;
                 handler.abortController = abortController;

From 759278d9a716df0d77e7526392f4ea90fe5d4d45 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 8 Nov 2021 23:25:12 +0200
Subject: [PATCH 087/102] upgrade iconv-lite

---
 package.json |  2 +-
 yarn.lock    | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/package.json b/package.json
index 63742637d..239fa5d46 100644
--- a/package.json
+++ b/package.json
@@ -30,7 +30,7 @@
         "globby": "^12.0.2",
         "graceful-cluster": "0.0.3",
         "htmlparser2": "^7.1.2",
-        "iconv-lite": "0.4.17",
+        "iconv-lite": "^0.6.3",
         "jslint": "^0.12.1",
         "jsontoxml": "0.0.11",
         "memcached": "2.2.2",
diff --git a/yarn.lock b/yarn.lock
index df3e02b76..28a465e20 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1003,11 +1003,6 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-iconv-lite@0.4.17:
-  version "0.4.17"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.17.tgz#4fdaa3b38acbc2c031b045d0edcdfe1ecab18c8d"
-  integrity sha1-T9qjs4rLwsAxsEXQ7c3+HsqxjI0=
-
 iconv-lite@0.4.24:
   version "0.4.24"
   resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
@@ -1015,6 +1010,13 @@ iconv-lite@0.4.24:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
+iconv-lite@^0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501"
+  integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==
+  dependencies:
+    safer-buffer ">= 2.1.2 < 3.0.0"
+
 ieee754@^1.1.13:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
@@ -1828,7 +1830,7 @@ safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, s
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
-"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
+"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==

From 1440478987e1174dc84a5998588ec12618660556 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 9 Nov 2021 18:39:10 +0200
Subject: [PATCH 088/102] fetch: retry request on http1 when http2 error

---
 lib/fetch.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 3542893dc..748b8b414 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,4 +1,5 @@
 import { h1NoCache, noCache, createUrl, AbortController, AbortError } from '@adobe/helix-fetch';
+import log from '../logging.js';
 
 const fetchKeepAlive = noCache({
     h1: {
@@ -48,11 +49,16 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                 resolve(stream);
             })
             .catch(error => {
-                if (error instanceof AbortError) {
-                    // `AbortError` before `response` occurs only on timeout.
-                    error = 'timeout';
+                if (!options.disable_http2 && error.code && /^ERR_HTTP2/.test(error.code)) {
+                    log('   -- doFetch http2 error', error.code, uri);
+                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                } else {
+                    if (error instanceof AbortError) {
+                        // `AbortError` before `response` occurs only on timeout.
+                        error = 'timeout';
+                    }
+                    reject(error);
                 }
-                reject(error);
             })
             .finally(() => {
                 clearTimeout(timeoutTimerId);

From 00903fe0d18c10e24f137e0322a2972ba2928161 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 9 Nov 2021 21:47:51 +0200
Subject: [PATCH 089/102] return `response.abort()` logic

---
 lib/plugins/system/htmlparser/htmlparser.js | 9 +++++----
 lib/utils.js                                | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 175b65b4f..eab878169 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -85,8 +85,8 @@ export default {
                 const abortController = resp.abortController;
 
                 if (resp.status >= 300 && resp.status< 400 && headers.location) {
-                    // abortController.abort();
-                    resp.pause();
+                    abortController.abort();
+                    //resp.pause();
                     var redirectUrl = urlLib.resolve(url, headers.location);
                     // Prevent cache self redirect. Some sites changes cookies and stops redirect loop (e.g. https://miro.com/app/live-embed/o9J_lBwNMhI=/?embedAutoplay=true)
                     var preventCache = redirectUrl === url;
@@ -102,8 +102,8 @@ export default {
                 }
 
                 if (resp.status !== 200) {
-                    // abortController.abort();
-                    resp.pause();
+                    abortController.abort();
+                    //resp.pause();
                     if (!!options.exposeStatusCode) {
                         return cacheAndRespond(null, {
                             __statusCode: resp.status,
@@ -118,6 +118,7 @@ export default {
 
                 if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){
                     abortController.abort();
+                    //resp.pause();
                     return cacheAndRespond(null, {
                         __nonHtmlContentData: {
                             type: headers['content-type'],
diff --git a/lib/utils.js b/lib/utils.js
index 333c59bcd..c927c2a15 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1016,7 +1016,7 @@ var getUriStatusPrivate = function(uri, options, cb) {
         fetchStream(request_options)
             .then(res => {
                 // TODO: may cause AbortError before cb.
-                //res.abortController.abort();
+                res.abortController.abort();
                 var data = {
                     code: res.status,
                     content_type: res.headers && res.headers['content-type'],

From cd1533030781ab6d20a055431714a1f6ab2c0a49 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 11 Nov 2021 00:04:21 +0200
Subject: [PATCH 090/102] fetch: fix `FetchError ABORT_ERR` when shared session
 aborted for specific domains

---
 lib/fetch.js                                | 22 +++++++++++++++------
 lib/plugins/system/htmlparser/htmlparser.js |  5 ++---
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 748b8b414..d8a7b435a 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,4 +1,4 @@
-import { h1NoCache, noCache, createUrl, AbortController, AbortError } from '@adobe/helix-fetch';
+import { h1NoCache, noCache, createUrl, AbortController, AbortError, FetchError } from '@adobe/helix-fetch';
 import log from '../logging.js';
 
 const fetchKeepAlive = noCache({
@@ -33,7 +33,7 @@ function doFetch(fetch_func, h1_fetch_func, options) {
     fetch_options.signal = abortController.signal;
     // Implement `timeout`.
     const timeoutTimerId = setTimeout(() => {
-        abortController.abort()
+        abortController.abort();
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
     // Implement `qs` (get params).
     const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
@@ -53,11 +53,21 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                     log('   -- doFetch http2 error', error.code, uri);
                     resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
                 } else {
-                    if (error instanceof AbortError) {
-                        // `AbortError` before `response` occurs only on timeout.
-                        error = 'timeout';
+                    if (!options.disable_http2 && error.code && error instanceof FetchError && error.code === 'ABORT_ERR') {
+                        // Special case, when shared session request aborted by htmlparser logic.
+                        /**
+                         * https://polldaddy.com/poll/7451882/?s=twitter
+                         * https://app.everviz.com/show/O0Cy7Dyt
+                         */
+                        log('   -- doFetch h2 aborted error', uri);
+                        resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                    } else {
+                        if (error instanceof AbortError) {
+                            // `AbortError` before `response` occurs only on timeout.
+                            error = 'timeout';
+                        }
+                        reject(error);
                     }
-                    reject(error);
                 }
             })
             .finally(() => {
diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index eab878169..104e8df6d 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -85,8 +85,8 @@ export default {
                 const abortController = resp.abortController;
 
                 if (resp.status >= 300 && resp.status< 400 && headers.location) {
+                    // TODO: do we need abort if no content?
                     abortController.abort();
-                    //resp.pause();
                     var redirectUrl = urlLib.resolve(url, headers.location);
                     // Prevent cache self redirect. Some sites changes cookies and stops redirect loop (e.g. https://miro.com/app/live-embed/o9J_lBwNMhI=/?embedAutoplay=true)
                     var preventCache = redirectUrl === url;
@@ -102,8 +102,8 @@ export default {
                 }
 
                 if (resp.status !== 200) {
+                    // TODO: do we need abort if no content?
                     abortController.abort();
-                    //resp.pause();
                     if (!!options.exposeStatusCode) {
                         return cacheAndRespond(null, {
                             __statusCode: resp.status,
@@ -118,7 +118,6 @@ export default {
 
                 if('content-type' in headers && !/text\/html|application\/xhtml\+xml/gi.test(headers['content-type'])){
                     abortController.abort();
-                    //resp.pause();
                     return cacheAndRespond(null, {
                         __nonHtmlContentData: {
                             type: headers['content-type'],

From 1ddef3788659d760ca2fc7c475cb4b0a69743200 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 11 Nov 2021 00:04:28 +0200
Subject: [PATCH 091/102] Update .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 108d58572..97dd27920 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ lib/new/test.js
 
 iframely-private-plugins/
 config.providers.js
+.vscode/launch.json

From 633678065d7e633acde377329a400e9416aa33a9 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Thu, 11 Nov 2021 00:17:37 +0200
Subject: [PATCH 092/102] htmlparser: do not abort on http error code

---
 lib/plugins/system/htmlparser/htmlparser.js | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js
index 104e8df6d..31c1c91d5 100644
--- a/lib/plugins/system/htmlparser/htmlparser.js
+++ b/lib/plugins/system/htmlparser/htmlparser.js
@@ -85,8 +85,6 @@ export default {
                 const abortController = resp.abortController;
 
                 if (resp.status >= 300 && resp.status< 400 && headers.location) {
-                    // TODO: do we need abort if no content?
-                    abortController.abort();
                     var redirectUrl = urlLib.resolve(url, headers.location);
                     // Prevent cache self redirect. Some sites changes cookies and stops redirect loop (e.g. https://miro.com/app/live-embed/o9J_lBwNMhI=/?embedAutoplay=true)
                     var preventCache = redirectUrl === url;
@@ -102,8 +100,6 @@ export default {
                 }
 
                 if (resp.status !== 200) {
-                    // TODO: do we need abort if no content?
-                    abortController.abort();
                     if (!!options.exposeStatusCode) {
                         return cacheAndRespond(null, {
                             __statusCode: resp.status,

From d81041c90eedeb0d6016c2f5f9de45fbce8b8a96 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 12 Nov 2021 17:09:58 +0200
Subject: [PATCH 093/102] fix config loading

---
 app.js                          | 2 ++
 config.loader.js                | 3 +++
 lib/cache-engines/memcached.js  | 2 +-
 lib/cache-engines/node-cache.js | 2 +-
 lib/cache-engines/redis.js      | 2 +-
 lib/cache.js                    | 2 +-
 lib/html-utils.js               | 2 +-
 lib/loader/pluginLoader.js      | 4 ++++
 lib/utils.js                    | 6 +-----
 lib/whitelist.js                | 2 ++
 modules/tests-ui/models.js      | 3 +--
 modules/tests-ui/tester.js      | 2 +-
 utils.js                        | 3 ---
 13 files changed, 19 insertions(+), 16 deletions(-)
 create mode 100644 config.loader.js

diff --git a/app.js b/app.js
index 75d0c509a..60f407f1e 100644
--- a/app.js
+++ b/app.js
@@ -1,4 +1,6 @@
 import { cacheMiddleware, NotFound } from './utils.js';
+import CONFIG from './config.loader.js';
+global.CONFIG = CONFIG;
 
 console.log("");
 console.log("Starting Iframely...");
diff --git a/config.loader.js b/config.loader.js
new file mode 100644
index 000000000..797244fb9
--- /dev/null
+++ b/config.loader.js
@@ -0,0 +1,3 @@
+var globalConfig = await import(process.cwd() + '/config.js');
+globalConfig = globalConfig && globalConfig.default;
+export default globalConfig;
\ No newline at end of file
diff --git a/lib/cache-engines/memcached.js b/lib/cache-engines/memcached.js
index 1e4c0d862..7105c9f79 100644
--- a/lib/cache-engines/memcached.js
+++ b/lib/cache-engines/memcached.js
@@ -1,7 +1,7 @@
 import log from '../../logging.js';
 import * as crypto from 'crypto';
 import Memcached from 'memcached';
-import CONFIG from '../../config.js';
+import CONFIG from '../../config.loader.js';
 
 var memcached = new Memcached(CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.locations, CONFIG.MEMCACHED_OPTIONS && CONFIG.MEMCACHED_OPTIONS.options);
 
diff --git a/lib/cache-engines/node-cache.js b/lib/cache-engines/node-cache.js
index ea109dc10..823b21a15 100644
--- a/lib/cache-engines/node-cache.js
+++ b/lib/cache-engines/node-cache.js
@@ -1,5 +1,5 @@
     import NodeCache from "node-cache";
-    import CONFIG from '../../config.js';
+    import CONFIG from '../../config.loader.js';
 
     var nodeCache = new NodeCache({
         stdTTL: CONFIG.CACHE_TTL,
diff --git a/lib/cache-engines/redis.js b/lib/cache-engines/redis.js
index dd91214d9..8b03b1dd6 100644
--- a/lib/cache-engines/redis.js
+++ b/lib/cache-engines/redis.js
@@ -1,5 +1,5 @@
     import log from '../../logging.js';
-    import CONFIG from '../../config.js';
+    import CONFIG from '../../config.loader.js';
 
     var client;
 
diff --git a/lib/cache.js b/lib/cache.js
index 26ac19332..470e4a49d 100644
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -1,5 +1,5 @@
     import * as _ from 'underscore';
-    import CONFIG from '../config.js';
+    import CONFIG from '../config.loader.js';
 
     var DEFAULT_CACHE = "node-cache";
 
diff --git a/lib/html-utils.js b/lib/html-utils.js
index e8a44c0e9..b2a73b107 100644
--- a/lib/html-utils.js
+++ b/lib/html-utils.js
@@ -1,7 +1,7 @@
     import cheerio from 'cheerio';
     
     import * as _ from 'underscore';
-    import CONFIG from '../config.js';
+    import CONFIG from '../config.loader.js';
 
     var defaultPaddingBottom = 100 / CONFIG.DEFAULT_ASPECT_RATIO;
 
diff --git a/lib/loader/pluginLoader.js b/lib/loader/pluginLoader.js
index 2a3ffc86f..0ce66a98e 100644
--- a/lib/loader/pluginLoader.js
+++ b/lib/loader/pluginLoader.js
@@ -6,6 +6,10 @@
     var JSLINT = node_jslint.load('latest');
     import * as utils from './utils.js';
 
+    import CONFIG from '../../config.loader.js';
+    // Global CONFIG used by plugins loaded during module import.
+    global.CONFIG = CONFIG;
+
     var PLUGIN_METHODS = utils.PLUGIN_METHODS,
         PLUGINS_FIELDS = utils.PLUGIN_FIELDS,
         PLUGIN_FIELDS_TYPE_DICT = utils.PLUGIN_FIELDS_TYPE_DICT,
diff --git a/lib/utils.js b/lib/utils.js
index c927c2a15..595148192 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -10,11 +10,7 @@ import { cache } from './cache.js';
 import * as htmlUtils from './html-utils.js';
 import log from '../logging.js';
 
-if (!global.CONFIG) {
-    // TODO: review
-     var config = await import('../config.js');
-     global.CONFIG = config.default;
-}
+import CONFIG from '../config.loader.js';
 
 function prepareEncodedUri(request_options, attr) {
     var url = request_options[attr];
diff --git a/lib/whitelist.js b/lib/whitelist.js
index 20774a649..ed01ec165 100644
--- a/lib/whitelist.js
+++ b/lib/whitelist.js
@@ -10,6 +10,8 @@
     import { fileURLToPath } from 'url';
     import { dirname } from 'path';
 
+    import CONFIG from '../config.loader.js';
+
     const __filename = fileURLToPath(import.meta.url);
     const __dirname = dirname(__filename);
 
diff --git a/modules/tests-ui/models.js b/modules/tests-ui/models.js
index f54a08f23..46f00f783 100644
--- a/modules/tests-ui/models.js
+++ b/modules/tests-ui/models.js
@@ -1,7 +1,6 @@
     import moment from 'moment';
     import mongoose from 'mongoose';
-    import config from '../../config.js';
-    global.CONFIG = config;
+    import CONFIG from '../../config.loader.js';
 
     mongoose.set('useUnifiedTopology', true);
     mongoose.set('useCreateIndex', true);
diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js
index 6fd6a1546..90fd583c6 100644
--- a/modules/tests-ui/tester.js
+++ b/modules/tests-ui/tester.js
@@ -1,4 +1,4 @@
-import config from '../../config.js';
+import CONFIG from '../../config.loader.js';
 global.CONFIG = config;
 
 if (!CONFIG.tests) {
diff --git a/utils.js b/utils.js
index cd1f2e90b..0a31c315b 100644
--- a/utils.js
+++ b/utils.js
@@ -1,6 +1,3 @@
-    import config from './config.js';
-    global.CONFIG = config;
-
     import * as async from 'async';
     import { cache } from './lib/cache.js';
     import * as ejs from 'ejs';

From 0b0b8a09e7ae55830153d35dd11a328fa3a44bb7 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 12 Nov 2021 17:48:02 +0200
Subject: [PATCH 094/102] config comment

---
 config.loader.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config.loader.js b/config.loader.js
index 797244fb9..261372d7d 100644
--- a/config.loader.js
+++ b/config.loader.js
@@ -1,3 +1,4 @@
+// Load global config from exec dir, because `iframely` can be used as library.
 var globalConfig = await import(process.cwd() + '/config.js');
 globalConfig = globalConfig && globalConfig.default;
 export default globalConfig;
\ No newline at end of file

From 0aff9404bc937d88420d8b25eefd71affaf8396d Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Fri, 12 Nov 2021 18:14:16 +0200
Subject: [PATCH 095/102] config: always use `iframely` config;  use spread to
 merge objects

---
 config.loader.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config.loader.js b/config.loader.js
index 261372d7d..e7bf4c1a4 100644
--- a/config.loader.js
+++ b/config.loader.js
@@ -1,4 +1,5 @@
+import iframelyConfig from './config.js';
 // Load global config from exec dir, because `iframely` can be used as library.
 var globalConfig = await import(process.cwd() + '/config.js');
 globalConfig = globalConfig && globalConfig.default;
-export default globalConfig;
\ No newline at end of file
+export default {...iframelyConfig, ...globalConfig};
\ No newline at end of file

From d9e7500575caefa0ca5603ffcf3679c4c8c50ec6 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 15 Nov 2021 19:13:51 +0200
Subject: [PATCH 096/102] fetch: remove hash part of url

---
 lib/fetch.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index d8a7b435a..4436cde2c 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -36,7 +36,9 @@ function doFetch(fetch_func, h1_fetch_func, options) {
         abortController.abort();
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
     // Implement `qs` (get params).
-    const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
+    var uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
+    // Remove hash part of url.
+    uri = uri.replace(/#.*/gi, '');
     const a_fetch_func = options.disable_http2 ? h1_fetch_func: fetch_func;
     return new Promise((resolve, reject) => {
         a_fetch_func(uri, fetch_options)

From ee3751fd75bb0a66b9f42005bd0d40570ac952ec Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 16 Nov 2021 22:56:06 +0200
Subject: [PATCH 097/102] fetch: HostAbortController to prevent abort opened
 connections on host

---
 lib/fetch.js | 93 ++++++++++++++++++++++++++++++++++++++++++++++++----
 lib/utils.js |  5 ++-
 2 files changed, 91 insertions(+), 7 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 4436cde2c..8297f7b09 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -1,3 +1,4 @@
+import { URL } from 'url';
 import { h1NoCache, noCache, createUrl, AbortController, AbortError, FetchError } from '@adobe/helix-fetch';
 import log from '../logging.js';
 
@@ -27,23 +28,29 @@ const fetchH1 = h1NoCache({
 }).fetch;
 
 function doFetch(fetch_func, h1_fetch_func, options) {
-    const abortController = new AbortController();
+    
     const fetch_options = Object.assign({}, options);
+    
+    // Implement `qs` (get params).
+    var uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
+    // Remove hash part of url.
+    uri = uri.replace(/#.*/gi, '');
+
+    const abortController = new HostAbortController(uri);
+
     // Allow request abort before finish.
     fetch_options.signal = abortController.signal;
     // Implement `timeout`.
     const timeoutTimerId = setTimeout(() => {
         abortController.abort();
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
-    // Implement `qs` (get params).
-    var uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
-    // Remove hash part of url.
-    uri = uri.replace(/#.*/gi, '');
+
     const a_fetch_func = options.disable_http2 ? h1_fetch_func: fetch_func;
     return new Promise((resolve, reject) => {
         a_fetch_func(uri, fetch_options)
             .then(response => {
                 var stream = response.body;
+                abortController.onResponse(stream);
                 stream.status = response.status;
                 stream.headers = response.headers.plain();
                 stream.abortController = abortController;
@@ -90,6 +97,7 @@ export function fetchStreamAuthorized(options) {
     return doFetch(fetchAuthorized, fetchH1Authorized, options);
 };
 
+// TODO: abort controller?
 export function fetchData(options) {
     var json = options.json;
     delete options.json;
@@ -120,4 +128,77 @@ export function fetchData(options) {
             })
             .catch(reject);
     });
-};
\ No newline at end of file
+};
+
+const hostsCache = {};
+
+function addController(ctrl) {
+    hostsCache[ctrl.host] = hostsCache[ctrl.host] || [];
+    hostsCache[ctrl.host].push(ctrl);
+}
+
+function tryAbortHost(host) {
+    var controllers = hostsCache[host];
+    if (controllers) {
+        const hasWaitingRequests = controllers.some(ctrl => ctrl.waiting);
+        // If all aborted or finished.
+        if (!hasWaitingRequests) {
+            while (controllers.length) {
+                let ctrl = controllers.pop();
+                ctrl.forceAbort();
+            }
+        }
+    }
+}
+
+function removeController(ctrl) {
+    var controllers = hostsCache[ctrl.host];
+    if (controllers) {
+        const idx = controllers.indexOf(ctrl);
+        controllers.splice(idx, 1);
+    }
+}
+
+class HostAbortController {
+
+    constructor(url) {
+        this.aborted = false;
+        this.responded = false;
+        this.url = url;
+        this.host = new URL(url).hostname;
+        this.abortController = new AbortController();
+        addController(this);
+    }
+
+    onResponse(stream) {
+        this.stream = stream;
+        if (this.aborted) {
+            stream.pause();
+        }
+        stream.on('end', () => {
+            this.finished = true;
+            removeController(this);
+            tryAbortHost(this.host);
+        });
+    }
+
+    abort() {
+        this.stream && this.stream.pause();
+        this.aborted = true;
+        tryAbortHost(this.host);
+    }
+
+    forceAbort() {
+        if (this.aborted && !this.finished) {
+            this.abortController.abort();
+        }
+    }
+
+    get waiting() {
+        return !(this.aborted || this.finished);
+    }
+
+    get signal() {
+        return this.abortController.signal;
+    }
+}
\ No newline at end of file
diff --git a/lib/utils.js b/lib/utils.js
index 595148192..df13b3d2c 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -627,7 +627,10 @@ export function getContentType(uriForCache, uriOriginal, options, cb) {
             }, {
                 onResponse: function(res) {
 
-                    abortController = res.abortController;
+                    // Skip abort for head.
+                    if (methodCaller === getUrlFunctional) {
+                        abortController = res.abortController;
+                    }
 
                     var error = res.status && res.status != 200 ? res.status : null;
 

From 987c8c865138404c3e10f0feff10eab0e8769280 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 16 Nov 2021 23:44:32 +0200
Subject: [PATCH 098/102] fetch: use timeout for response data

---
 lib/fetch.js | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index 8297f7b09..a545ddabe 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -50,6 +50,9 @@ function doFetch(fetch_func, h1_fetch_func, options) {
         a_fetch_func(uri, fetch_options)
             .then(response => {
                 var stream = response.body;
+                stream.on('end', () => {
+                    clearTimeout(timeoutTimerId);
+                });
                 abortController.onResponse(stream);
                 stream.status = response.status;
                 stream.headers = response.headers.plain();
@@ -58,6 +61,7 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                 resolve(stream);
             })
             .catch(error => {
+                clearTimeout(timeoutTimerId);
                 if (!options.disable_http2 && error.code && /^ERR_HTTP2/.test(error.code)) {
                     log('   -- doFetch http2 error', error.code, uri);
                     resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
@@ -78,9 +82,6 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                         reject(error);
                     }
                 }
-            })
-            .finally(() => {
-                clearTimeout(timeoutTimerId);
             });
     });
 }
@@ -101,14 +102,30 @@ export function fetchStreamAuthorized(options) {
 export function fetchData(options) {
     var json = options.json;
     delete options.json;
-    var response;
+    var res;
     const fetch_options = Object.assign({}, options);
     const uri = options.qs ? createUrl(options.uri, options.qs) : options.uri;
+
+    const abortController = new HostAbortController(uri);
+
+    // Allow request abort before finish.
+    fetch_options.signal = abortController.signal;
+    // Implement `timeout`.
+    const timeoutTimerId = setTimeout(() => {
+        abortController.abort();
+    }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
+
     const a_fetch_func = options.disable_http2 ? fetchH1: fetch;
     return new Promise((resolve, reject) => {
         a_fetch_func(uri, fetch_options)
-            .then(res => {
-                response = res;
+            .then(response => {
+                var stream = response.body;
+                // TODO: looks like HEAD request has no END event.
+                stream.on('end', () => {
+                    clearTimeout(timeoutTimerId);
+                });
+                abortController.onResponse(stream);
+                res = response;
                 if (json !== false) {
                     // If `json` not forbidden, read `content-type`.
                     json = json || (response.headers.get('content-type').indexOf('application/json') > -1);
@@ -121,12 +138,15 @@ export function fetchData(options) {
             })
             .then(data => {
                 resolve({
-                    status: response.status,
-                    headers: response.headers.plain(),
+                    status: res.status,
+                    headers: res.headers.plain(),
                     data: data
                 });
             })
-            .catch(reject);
+            .catch((error) => {
+                clearTimeout(timeoutTimerId);
+                reject(error);
+            });
     });
 };
 
@@ -165,7 +185,8 @@ class HostAbortController {
         this.aborted = false;
         this.responded = false;
         this.url = url;
-        this.host = new URL(url).hostname;
+        const parsedUrl = new URL(url);
+        this.host = parsedUrl.protocol + parsedUrl.hostname;
         this.abortController = new AbortController();
         addController(this);
     }

From 4f0b1d6a105944cf9176ae365fc8e90d6fcf9b31 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Tue, 16 Nov 2021 23:46:31 +0200
Subject: [PATCH 099/102] bugfix

---
 modules/tests-ui/tester.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/tests-ui/tester.js b/modules/tests-ui/tester.js
index 90fd583c6..01c7f2048 100644
--- a/modules/tests-ui/tester.js
+++ b/modules/tests-ui/tester.js
@@ -1,5 +1,5 @@
 import CONFIG from '../../config.loader.js';
-global.CONFIG = config;
+global.CONFIG = CONFIG;
 
 if (!CONFIG.tests) {
     console.error('Tests not started: CONFIG.tests not configured.');

From f14074ec2ea569e125e9913e4a3f8793f3584271 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Mon, 15 Nov 2021 23:47:16 +0200
Subject: [PATCH 100/102] core: make wider post plugins iteration context

---
 lib/core.js | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/lib/core.js b/lib/core.js
index 99c71cdb7..8e5e69afa 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -956,8 +956,31 @@
 
         // After new context received - launch link post plugins.
 
+        var iterationPluginContexts = {};
+
+        var hasData = false;
+        for(var i = 0; i < result.length && !hasData; i++) {
+            var r = result[i];
+            if (r.data && !r.error && r.method.name === 'getLink' || r.method.name === 'getLinks' || r.method.name === 'prepareLink') {
+                var links;
+                if (r.method.name === 'prepareLink') {
+                    links = r.data && r.data.addLink;
+                } else {
+                    links = r.data;
+                }
+                if (links) {
+                    hasData = true;;
+                }
+            }
+        }
+
+        if (hasData) {
+            runPostPluginsIterationCall('startIteration', iterationPluginContexts);
+        }
+
         for(var i = 0; i < result.length; i++) {
 
+
             var r = result[i];
 
             if (r.data && !r.error && r.method.name === 'getLink' || r.method.name === 'getLinks' || r.method.name === 'prepareLink') {
@@ -980,20 +1003,18 @@
 
                 links = _.compact(links);
 
-                var iterationPluginContexts = {};
-
-                runPostPluginsIterationCall('startIteration', iterationPluginContexts);
-
                 for(var j = 0; j < links.length; j++) {
                     var link = links[j];
                     allResults.links.push(link);
                     runPostPlugins(link, r, usedMethods, context, pluginsContexts, iterationPluginContexts, options, asyncMethodCb);
                 }
-
-                runPostPluginsIterationCall('finishIteration', iterationPluginContexts);
             }
         }
 
+        if (hasData) {
+            runPostPluginsIterationCall('finishIteration', iterationPluginContexts);
+        }
+
         return hasNewData;
     }
 

From 381bc0e437e31f85fe754abef267fe11e398641c Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 17 Nov 2021 17:54:30 +0200
Subject: [PATCH 101/102] domains: remove request `timeout: 1000`

---
 plugins/domains/sverigesradio.se.js      | 1 -
 plugins/domains/tumblr.com/tumblr.api.js | 1 -
 2 files changed, 2 deletions(-)

diff --git a/plugins/domains/sverigesradio.se.js b/plugins/domains/sverigesradio.se.js
index 63fc17a2e..82b15e34b 100644
--- a/plugins/domains/sverigesradio.se.js
+++ b/plugins/domains/sverigesradio.se.js
@@ -14,7 +14,6 @@ export default {
 
         request({
             uri: "http://sverigesradio.se/sida/playerajax/getaudiourl?id=" + urlMatch[1] + "&type=publication&quality=medium&format=iis",
-            timeout: 1000,
             prepareResult: function(error, response, body, cb) {
 
                 // 404 means article is not embeddable
diff --git a/plugins/domains/tumblr.com/tumblr.api.js b/plugins/domains/tumblr.com/tumblr.api.js
index 33f45a02f..a3a44edf1 100644
--- a/plugins/domains/tumblr.com/tumblr.api.js
+++ b/plugins/domains/tumblr.com/tumblr.api.js
@@ -75,7 +75,6 @@ export default {
                 id: urlMatch[3]
             },
             json: true,
-            timeout: 1000,
             prepareResult: function (error, response, body, cb) {
 
                 if (error || body.errors) {

From 63acdf0b361cd9c631b65808a050f4ffc3b46a79 Mon Sep 17 00:00:00 2001
From: Nazar Leush <n.leush@gmail.com>
Date: Wed, 17 Nov 2021 20:53:27 +0200
Subject: [PATCH 102/102] always abort on head request

---
 lib/fetch.js | 1 -
 lib/utils.js | 5 +----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/fetch.js b/lib/fetch.js
index a545ddabe..ab24f36f9 100644
--- a/lib/fetch.js
+++ b/lib/fetch.js
@@ -98,7 +98,6 @@ export function fetchStreamAuthorized(options) {
     return doFetch(fetchAuthorized, fetchH1Authorized, options);
 };
 
-// TODO: abort controller?
 export function fetchData(options) {
     var json = options.json;
     delete options.json;
diff --git a/lib/utils.js b/lib/utils.js
index df13b3d2c..595148192 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -627,10 +627,7 @@ export function getContentType(uriForCache, uriOriginal, options, cb) {
             }, {
                 onResponse: function(res) {
 
-                    // Skip abort for head.
-                    if (methodCaller === getUrlFunctional) {
-                        abortController = res.abortController;
-                    }
+                    abortController = res.abortController;
 
                     var error = res.status && res.status != 200 ? res.status : null;