Merge branch 'master' into keyPress

internal/chrome_desktop.py

@@ -68,6 +68,7 @@
     'MediaRouter',
     'OfflinePagesPrefetching',
     'OptimizationHints',
+    'SidePanelPinning',
     'Translate',
     # Disable noisy Edge features
     'msAutofillEdgeCoupons',

internal/devtools.py

@@ -1362,7 +1362,35 @@ def type_text(self, string):
                     self.send_character(char)
         except Exception:
             logging.exception('Error running type command')
+    def mouse_press(self, command_options):
+        """Press down the mouse"""
+        params = {
+            'type': 'mousePressed',
+            'x': command_options['x'],
+            'y': command_options['y'],
+            'button': command_options['button'],
+            'clickCount': command_options['clickCount']
+        }
+        self.send_command('Input.dispatchMouseEvent', params)
+
+    def mouse_release(self, command_options):
+        """Let up the mouse"""
+        self.send_command('Input.dispatchMouseEvent', {
+            'type': 'mouseReleased',
+            'x': command_options['x'],
+            'y': command_options['y'],
+            'button': command_options['button'],
+            'clickCount': command_options['clickCount']
+        })
 
+    def mouse_click(self, params):
+        """Simulate pressing the mouse"""
+        try:
+            self.mouse_press(params)
+            self.mouse_release(params)
+        except Exception:
+            logging.exception('Error running mouse click command')
+
     def enable_target(self, target_id=None):
         """Hook up the necessary network (or other) events for the given target"""
         try:

internal/devtools_browser.py

@@ -777,6 +777,43 @@ def process_command(self, command):
                 if keyModifier in KeyModifiers.keys():
                     modifier = KeyModifiers[keyModifier]
             self.devtools.keypress(command['target'], modifier)
+        elif command['command'] == 'mouseClick':
+            if 'target' in command:
+                target = command['target']
+                separator = target.find('=')
+                if separator == -1:
+                    separator = target.find("'")
+                if separator >= 0:
+                    attribute = target[:separator]
+                    attr_value = target[separator + 1:]
+                    try:
+                        query = "JSON.stringify(document.querySelector('[{0}=\"{1}\"]').getBoundingClientRect())".format(
+                            attribute, attr_value)
+                        resp = self.devtools.execute_js(query, use_execution_context = True)
+                        resp_json = json.loads(resp)
+
+                        value = command['value']
+                        button = 'left'
+                        clickCount = 1                 
+                        if value in ['left', 'right']:
+                            button = value
+                        elif value == 'double':
+                            clickCount = 2                 
+                        elif value is not None:
+                            logging.info("Click type is not defined.")
+
+                        if 'x' in resp_json and 'y' in resp_json and 'width' in resp_json and 'height' in resp_json:
+                            x = int(float(resp_json['x'])) + int(float(resp_json['width']))/2
+                            y = int(float(resp_json['y'])) + int(float(resp_json['height']))/2
+                            command_options = {}
+                            command_options['x'] = x
+                            command_options['y'] = y
+                            command_options['button'] = button
+                            command_options['clickCount'] = clickCount
+                            self.devtools.mouse_click(command_options)
+                    except:
+                        self.task['error'] = 'Exception parsing mouseClick arguments.'
+                        logging.error(self.task['error'])
         elif command['command'] == 'waitfor':
             try:
                 self.devtools.wait_for_script = command['target'] if command['target'] else None
@@ -865,7 +902,12 @@ def run_lighthouse_test(self, task):
             else:
                 cpu_throttle = '{:.3f}'.format(self.job['throttle_cpu']) if 'throttle_cpu' in self.job else '1'
                 if self.job['dtShaper']:
-                    command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '150', '--throttling.downloadThroughputKbps', '1600', '--throttling.uploadThroughputKbps', '768', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
+                    if self.options.android or ('mobile' in self.job and self.job['mobile']):
+                        # 1.6Mbps down, 750Kbps up, 150ms RTT
+                        command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '150', '--throttling.downloadThroughputKbps', '1600', '--throttling.uploadThroughputKbps', '750', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
+                    else:
+                        # 10Mbps, 40ms RTT
+                        command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '40', '--throttling.downloadThroughputKbps', '10240', '--throttling.uploadThroughputKbps', '10240', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
                 elif 'throttle_cpu_requested' in self.job and self.job['throttle_cpu_requested'] > 1:
                     command.extend(['--throttling-method', 'devtools', '--throttling.requestLatencyMs', '0', '--throttling.downloadThroughputKbps', '0', '--throttling.uploadThroughputKbps', '0', '--throttling.cpuSlowdownMultiplier', cpu_throttle])
                 else:

internal/firefox.py

@@ -31,6 +31,8 @@
     import json
 from .desktop_browser import DesktopBrowser
 
+def _get_location_uri(accuracy, lat, lng) -> str:
+    return f'data:application/json, {{ "status":"OK", "accuracy":{accuracy}, "location":{{ "lat":{lat}, "lng":{lng} }} }}'
 
 class Firefox(DesktopBrowser):
     """Firefox"""
@@ -140,26 +142,47 @@ def start_firefox(self, job, task):
             return
         from selenium import webdriver # pylint: disable=import-error
 
-        capabilities = webdriver.DesiredCapabilities.FIREFOX.copy()
-        if 'ignoreSSL' in job and job['ignoreSSL']:
-            capabilities['acceptInsecureCerts'] = True
-        else:
-            capabilities['acceptInsecureCerts'] = False
+        if webdriver.__version__ >= "4.12":
+            service_args = ["--marionette-port", "2828"]
+            service = webdriver.FirefoxService(service_args=service_args, log_output=os.environ["MOZ_LOG_FILE"])
 
-        capabilities['moz:firefoxOptions'] = {
-            'binary': self.path,
-            'args': ['-profile', task['profile']],
-            'prefs': self.prepare_prefs(),
-            "log": {"level": "error"},
-            'env': {
-                "MOZ_LOG_FILE": os.environ["MOZ_LOG_FILE"],
-                "MOZ_LOG": os.environ["MOZ_LOG"]
-            }
-        }
-        service_args = ["--marionette-port", "2828"]
+            options = webdriver.FirefoxOptions()
+            options.binary_location = self.path
+            options.add_argument('--profile')
+            options.add_argument(f'{task["profile"]}')
+            options.log.level = 'error'
+            options.prefs = self.prepare_prefs()
 
-        self.driver = webdriver.Firefox(desired_capabilities=capabilities, service_args=service_args)
-        logging.debug(self.driver.capabilities)
+            capabilities = webdriver.DesiredCapabilities.FIREFOX.copy()
+            if 'ignoreSSL' in job and job['ignoreSSL']:
+                capabilities['acceptInsecureCerts'] = True
+            else:
+                capabilities['acceptInsecureCerts'] = False
+
+            for key, value in capabilities.items():
+                options.set_capability(key, value)
+            self.driver = webdriver.Firefox(options=options, service=service)
+        elif webdriver.__version__ <= "4.9":
+            capabilities = webdriver.DesiredCapabilities.FIREFOX.copy()
+            if 'ignoreSSL' in job and job['ignoreSSL']:
+                capabilities['acceptInsecureCerts'] = True
+            else:
+                capabilities['acceptInsecureCerts'] = False
+
+            capabilities['moz:firefoxOptions'] = {
+                'binary': self.path,
+                'args': ['-profile', task['profile']],
+                'prefs': self.prepare_prefs(),
+                "log": {"level": "error"},
+                'env': {
+                    "MOZ_LOG_FILE": os.environ["MOZ_LOG_FILE"],
+                    "MOZ_LOG": os.environ["MOZ_LOG"]
+                }
+            }
+            service_args = ["--marionette-port", "2828"]
+            self.driver = webdriver.Firefox(desired_capabilities=capabilities, service_args=service_args)
+        else:
+            raise Exception("Unsupported selenium version %s", webdriver.__version__)
 
         self.driver.set_page_load_timeout(task['time_limit'])
         if 'browserVersion' in self.driver.capabilities:
@@ -208,17 +231,13 @@ def launch(self, job, task):
                     ua_string += ' ' + task['AppendUA']
                     modified = True
                 if modified:
-                    logging.debug(ua_string)
                     self.driver_set_pref('general.useragent.override', ua_string)
                 # Location
                 if 'lat' in self.job and 'lng' in self.job:
                     try:
                         lat = float(str(self.job['lat']))
                         lng = float(str(self.job['lng']))
-                        location_uri = 'data:application/json,{{'\
-                            '"status":"OK","accuracy":10.0,'\
-                            '"location":{{"lat":{0:f},"lng":{1:f}}}'\
-                            '}}'.format(lat, lng)
+                        location_uri = _get_location_uri(10, lat, lng)
                         logging.debug('Setting location: %s', location_uri)
                         self.driver_set_pref('geo.wifi.uri', location_uri)
                     except Exception:
@@ -261,20 +280,12 @@ def driver_set_pref(self, key, value):
         """Set a Firefox pref at runtime"""
         if self.driver is not None:
             try:
-                script = 'const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");'
-                script += 'Services.prefs.'
-                if isinstance(value, bool):
-                    script += 'setBoolPref'
-                elif isinstance(value, (str, unicode)):
-                    script += 'setStringPref'
-                else:
-                    script += 'setIntPref'
-                script += '({0}, {1});'.format(json.dumps(key), json.dumps(value))
-                logging.debug(script)
+                script = 'const { Preferences } = ChromeUtils.importESModule("resource://gre/modules/Preferences.sys.mjs");'
+                script += f'Preferences.set({json.dumps(key)}, {json.dumps(value)});'
                 self.driver.set_context(self.driver.CONTEXT_CHROME)
                 self.driver.execute_script(script)
-            except Exception:
-                logging.exception("Error setting pref")
+            except Exception as err:
+                logging.exception("Error setting pref %s => %s: %s", key, value, err)
             finally:
                 self.driver.set_context(self.driver.CONTEXT_CONTENT)
 
@@ -300,6 +311,8 @@ def close_browser(self, job, task):
         if platform.system() == "Linux":
             subprocess.call(['killall', '-9', 'firefox'])
             subprocess.call(['killall', '-9', 'firefox-trunk'])
+            subprocess.call(['killall', '-9', 'firefox-nightly'])
+            subprocess.call(['killall', '-9', 'firefox-esr'])
         os.environ["MOZ_LOG_FILE"] = ''
         os.environ["MOZ_LOG"] = ''
 
@@ -334,7 +347,7 @@ def run_axe(self, task):
                 script += "'" + "', '".join(axe_cats) + "'"
                 script += ']}).then(results=>{return results;});'
         except Exception as err:
-            logging.exception("Exception running Axe: %s", err.__str__())
+            logging.exception("Exception running Axe: %s", err)
         if self.must_exit_now:
             return
         completed = False
@@ -357,7 +370,7 @@ def run_axe(self, task):
                         axe_info['incomplete'] = axe_results['incomplete']
                     task['page_data']['axe'] = axe_info
         except Exception as err:
-            logging.exception("Exception running Axe: %s", err.__str__())
+            logging.exception("Exception running Axe: %s", err)
         if not completed:
             task['page_data']['axe_failed'] = 1
         self.axe_time = monotonic() - start
@@ -384,7 +397,7 @@ def run_task(self, task):
                     logging.exception("Exception running task")
                 if command['record']:
                     self.wait_for_page_load()
-                    if not task['combine_steps'] or not len(task['script']):
+                    if not task['combine_steps'] or not task['script']:
                         self.on_stop_capture(task)
                         self.on_stop_recording(task)
                         recording = False
@@ -405,10 +418,9 @@ def run_task(self, task):
             self.task = None
 
     def alert_size(self, _alert_config, _task_dir, _prefix):
-        '''Checks the agents file size and alert on certain percentage over avg byte size'''             
+        '''Checks the agents file size and alert on certain percentage over avg byte size'''
         self.alert_desktop_results(_alert_config, 'Firefox', _task_dir, _prefix)
 
-
     def wait_for_extension(self):
         """Wait for the extension to send the started message"""
         if self.job['message_server'] is not None:
@@ -514,7 +526,7 @@ def run_js_file(self, file_name):
         script = None
         script_file_path = os.path.join(self.script_dir, file_name)
         if os.path.isfile(script_file_path):
-            with open(script_file_path, 'r') as script_file:
+            with open(script_file_path, 'r', encoding='utf-8') as script_file:
                 script = script_file.read()
         if self.driver is not None and script is not None:
             try:
@@ -526,7 +538,7 @@ def run_js_file(self, file_name):
                 logging.debug(ret)
         return ret
 
-    def get_sorted_requests_json(self, include_bodies):
+    def get_sorted_requests_json(self, _include_bodies):
         return 'null'
 
     def collect_browser_metrics(self, task):
@@ -970,10 +982,7 @@ def process_command(self, command):
                     parts = command['target'].split(',')
                     lat = float(parts[0])
                     lng = float(parts[1])
-                    location_uri = 'data:application/json,{{'\
-                        '"status":"OK","accuracy":{2:d},'\
-                        '"location":{{"lat":{0:f},"lng":{1:f}}}'\
-                        '}}'.format(lat, lng, accuracy)
+                    location_uri = _get_location_uri(accuracy, lat, lng)
                     logging.debug('Setting location: %s', location_uri)
                     self.set_pref('geo.wifi.uri', location_uri)
             except Exception:

internal/optimization_checks.py

@@ -304,6 +304,10 @@ def __init__(self, job, task, requests):
                         '.cdn.dnsv1.com.cn',
                         '.dsa.dnsv1.com',
                         '.dsa.dnsv1.com.cn'],
+            'Transparent Edge': ['.edge2befaster.io',
+                                 '.edge2befaster.net',
+                                 '.edgetcdn.io',
+                                 '.edgetcdn.net'],
             'TRBCDN': ['.trbcdn.net'],
             'Twitter': ['.twimg.com'],
             'UnicornCDN': ['.unicorncdn.net'],

internal/support/firefox_log_parser.py

@@ -296,6 +296,14 @@ def socket_thread_http_entry(self, msg):
                 socket = self.http['current_socket']
                 self.http['connections'][connection] = {'socket': socket}
             del self.http['current_socket']
+        elif msg['message'].startswith('TlsHandshaker::SetupSSL '):
+            match = re.search(r'^TlsHandshaker::SetupSSL (?P<connection>[\w\d]+)',
+                              msg['message'])
+            if match:
+                connection = match.groupdict().get('connection')
+                if connection in self.http['connections']:
+                    if 'ssl_start' not in self.http['connections'][connection]:
+                        self.http['connections'][connection]['ssl_start'] = msg['timestamp']
         elif msg['message'].startswith('nsHttpConnection::SetupSSL '):
             match = re.search(r'^nsHttpConnection::SetupSSL (?P<connection>[\w\d]+)',
                               msg['message'])
@@ -332,6 +340,17 @@ def socket_thread_http_entry(self, msg):
                 if byte_count > 0 and trans_id in self.http['requests'] and \
                         'start' not in self.http['requests'][trans_id]:
                     self.http['requests'][trans_id]['start'] = msg['timestamp']
+        elif msg['message'].startswith('nsHttpTransaction::OnSocketStatus ') and \
+                msg['message'].find(' status=4b0005 progress=') > -1:
+            match = re.search(r'^nsHttpTransaction::OnSocketStatus '
+                              r'\[this=(?P<id>[\w\d]+) status=4b0005 progress=(?P<bytes>[\d+]+)',
+                              msg['message'])
+            if match:
+                trans_id = match.groupdict().get('id')
+                byte_count = int(match.groupdict().get('bytes'))
+                if byte_count > 0 and trans_id in self.http['requests'] and \
+                        'start' not in self.http['requests'][trans_id]:
+                    self.http['requests'][trans_id]['start'] = msg['timestamp']
         elif msg['message'].startswith('nsHttpTransaction::ProcessData '):
             match = re.search(r'^nsHttpTransaction::ProcessData \[this=(?P<id>[\w\d]+)',
                               msg['message'])
@@ -446,14 +465,15 @@ def socket_transport_entry(self, msg):
                 port = match.groupdict().get('port')
                 self.http['sockets'][socket] = {'host': host, 'port': port}
         # nsSocketTransport::SendStatus [this=143f4000 status=804b0007]
+        # nsSocketTransport::SendStatus [this=7fe074bd2a00 status=4B0007]
         elif msg['message'].startswith('nsSocketTransport::SendStatus '):
             match = re.search(r'^nsSocketTransport::SendStatus \['
                               r'this=(?P<socket>[\w\d]+) '
                               r'status=(?P<status>[\w\d]+)', msg['message'])
             if match:
                 socket = match.groupdict().get('socket')
                 status = match.groupdict().get('status')
-                if status == '804b0007':
+                if status in ['804b0007', '4b0007']:
                     if socket not in self.http['sockets']:
                         self.http['sockets'][socket] = {}
                     if 'start' not in self.http['sockets'][socket]: