-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/master'
- Loading branch information
Showing
14 changed files
with
885 additions
and
697 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,117 +1,55 @@ | ||
| from benchmark.test_types.framework_test_type import FrameworkTestType | ||
| from benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object | ||
|
|
||
| import json | ||
|
|
||
| class DBTestType(FrameworkTestType): | ||
| def __init__(self): | ||
| args = ['db_url'] | ||
| FrameworkTestType.__init__(self, name='db', | ||
| accept_header=self.accept_json, | ||
| requires_db=True, args=args) | ||
|
|
||
| def get_url(self): | ||
| return self.db_url | ||
|
|
||
| def verify(self, base_url): | ||
| '''Ensures body is valid JSON with a key 'id' and a key | ||
| 'randomNumber', both of which must map to integers | ||
| ''' | ||
|
|
||
| url = base_url + self.db_url | ||
| full_response = self._curl(url) | ||
| body = self._curl_body(url) | ||
|
|
||
| # Empty response | ||
| if body is None: | ||
| return [('fail','No response', url)] | ||
| elif len(body) == 0: | ||
| return [('fail','Empty Response', url)] | ||
|
|
||
| # Valid JSON? | ||
| try: | ||
| response = json.loads(body) | ||
| except ValueError as ve: | ||
| return [('fail',"Invalid JSON - %s" % ve, url)] | ||
|
|
||
| problems = [] | ||
|
|
||
| # We are allowing the single-object array | ||
| # e.g. [{'id':5, 'randomNumber':10}] for now, | ||
| # but will likely make this fail at some point | ||
| if type(response) == list: | ||
| response = response[0] | ||
| problems.append( ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url) ) | ||
|
|
||
| # Make sure there was a JSON object inside the array | ||
| if type(response) != dict: | ||
| problems.append( ('fail', 'Response is not a JSON object or an array of JSON objects', url) ) | ||
| return problems | ||
|
|
||
| problems += self._verifyObject(response, url) | ||
|
|
||
| # Ensure required response headers are present | ||
| if any(v.lower() not in full_response.lower() for v in ('Server','Date','Content-Type: application/json')) \ | ||
| or all(v.lower() not in full_response.lower() for v in ('Content-Length','Transfer-Encoding')): | ||
| problems.append( ('warn','Required response header missing.',url) ) | ||
|
|
||
| if len(problems) == 0: | ||
| return [('pass','',url)] | ||
| else: | ||
| return problems | ||
|
|
||
| def _verifyObject(self, db_object, url, max_infraction='fail'): | ||
| '''Ensure the passed item is a JSON object with | ||
| keys 'id' and 'randomNumber' mapping to ints. | ||
| Separate method allows the QueryTestType to | ||
| reuse these checks''' | ||
|
|
||
| problems = [] | ||
|
|
||
| if type(db_object) != dict: | ||
| got = str(db_object)[:20] | ||
| if len(str(db_object)) > 20: | ||
| got = str(db_object)[:17] + '...' | ||
| return [(max_infraction, "Expected a JSON object, got '%s' instead" % got, url)] | ||
|
|
||
| # Make keys case insensitive | ||
| db_object = {k.lower(): v for k,v in db_object.iteritems()} | ||
|
|
||
| if "id" not in db_object: | ||
| problems.append( (max_infraction, "Response has no 'id' key", url) ) | ||
| if "randomnumber" not in db_object: | ||
| problems.append( (max_infraction, "Response has no 'randomNumber' key", url) ) | ||
|
|
||
| # Ensure we can continue on to use these keys | ||
| if "id" not in db_object or "randomnumber" not in db_object: | ||
| return problems | ||
|
|
||
| try: | ||
| float(db_object["id"]) | ||
| except ValueError as ve: | ||
| problems.append( (max_infraction, "Response key 'id' does not map to a number - %s" % ve, url) ) | ||
|
|
||
| try: | ||
| float(db_object["randomnumber"]) | ||
| except ValueError as ve: | ||
| problems.append( (max_infraction, "Response key 'randomNumber' does not map to a number - %s" % ve, url) ) | ||
|
|
||
| if type(db_object["id"]) != int: | ||
| problems.append( ('warn', '''Response key 'id' contains extra quotations or decimal points. | ||
| This may negatively affect performance during benchmarking''', url) ) | ||
|
|
||
| # Tests based on the value of the numbers | ||
| try: | ||
| response_id = float(db_object["id"]) | ||
| response_rn = float(db_object["randomnumber"]) | ||
|
|
||
| if response_id > 10000 or response_id < 1: | ||
| problems.append( ('warn', "Response key 'id' should be between 1 and 10,000", url) ) | ||
|
|
||
| if response_rn > 10000: | ||
| problems.append( ('warn', '''Response key 'randomNumber' is over 10,000. This may negatively | ||
| afect performance by sending extra bytes.''', url) ) | ||
| except ValueError: | ||
| pass | ||
|
|
||
| return problems | ||
| class DBTestType(FrameworkTestType): | ||
|
|
||
| def __init__(self): | ||
| kwargs = { | ||
| 'name': 'db', | ||
| 'accept_header': self.accept('json'), | ||
| 'requires_db': True, | ||
| 'args': ['db_url'] | ||
| } | ||
| FrameworkTestType.__init__(self, **kwargs) | ||
|
|
||
| def get_url(self): | ||
| return self.db_url | ||
|
|
||
| def verify(self, base_url): | ||
| '''Ensures body is valid JSON with a key 'id' and a key | ||
| 'randomNumber', both of which must map to integers | ||
| ''' | ||
|
|
||
| url = base_url + self.db_url | ||
| headers, body = self.request_headers_and_body(url) | ||
|
|
||
| response, problems = basic_body_verification(body) | ||
|
|
||
| if len(problems) > 0: | ||
| return problems | ||
|
|
||
| # We are allowing the single-object array | ||
| # e.g. [{'id':5, 'randomNumber':10}] for now, | ||
| # but will likely make this fail at some point | ||
| if type(response) == list: | ||
| response = response[0] | ||
| problems.append( | ||
| ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url)) | ||
|
|
||
| # Make sure there was a JSON object inside the array | ||
| if type(response) != dict: | ||
| problems.append( | ||
| ('fail', 'Response is not a JSON object or an array of JSON objects', url)) | ||
| return problems | ||
|
|
||
| # Verify response content | ||
| problems += verify_randomnumber_object(response, url) | ||
| problems += verify_headers(headers, url, should_be='json') | ||
|
|
||
| if len(problems) == 0: | ||
| return [('pass', '', url)] | ||
| else: | ||
| return problems |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,71 +1,89 @@ | ||
| from benchmark.test_types.framework_test_type import FrameworkTestType | ||
| from benchmark.fortune_html_parser import FortuneHTMLParser | ||
| from benchmark.test_types.verifications import basic_body_verification, verify_headers | ||
|
|
||
|
|
||
| class FortuneTestType(FrameworkTestType): | ||
| def __init__(self): | ||
| args = ['fortune_url'] | ||
| FrameworkTestType.__init__(self, name='fortune', | ||
| requires_db=True, | ||
| accept_header=self.accept_html, | ||
| args=args) | ||
|
|
||
| def get_url(self): | ||
| return self.fortune_url | ||
|
|
||
| def verify(self, base_url): | ||
| '''Parses the given HTML string and asks the | ||
| FortuneHTMLParser whether the parsed string is a | ||
| valid fortune response | ||
| ''' | ||
| url = base_url + self.fortune_url | ||
| full_response = self._curl(url) | ||
| body = self._curl_body(url) | ||
|
|
||
| # Empty response | ||
| if body is None: | ||
| return [('fail','No response', url)] | ||
| elif len(body) == 0: | ||
| return [('fail','Empty Response', url)] | ||
|
|
||
| parser = FortuneHTMLParser() | ||
| parser.feed(body) | ||
| (valid, diff) = parser.isValidFortune(self.out) | ||
| if valid: | ||
| # Ensure required response headers are present | ||
| if any(v.lower() not in full_response.lower() for v in ('Server','Date','Content-Type: text/html')) \ | ||
| or all(v.lower() not in full_response.lower() for v in ('Content-Length','Transfer-Encoding')): | ||
| return[('warn','Required response header missing.',url)] | ||
|
|
||
| return [('pass','',url)] | ||
| else: | ||
| failures = [('fail','Invalid according to FortuneHTMLParser',url)] | ||
| # Catch exceptions because we are relying on internal code | ||
| try: | ||
| # Parsing this: | ||
| # --- Valid | ||
| # +++ Response | ||
| # @@ -1 +1 @@ | ||
| # | ||
| # -<!doctype html><html><head><title>Fortunes</title></head><body><table> | ||
| # +<!doctype html><html><head><meta></meta><title>Fortunes</title></head><body><div><table> | ||
| # @@ -16 +16 @@ | ||
|
|
||
| current_neg = [] | ||
| current_pos = [] | ||
| for line in diff[3:]: | ||
| if line[0] == '+': | ||
| current_neg.append(line[1:]) | ||
| elif line[0] == '-': | ||
| current_pos.append(line[1:]) | ||
| elif line[0] == '@': | ||
| failures.append( ('fail', | ||
| "`%s` should be `%s`" % (''.join(current_neg), ''.join(current_pos)), | ||
| url) ) | ||
| if len(current_pos) != 0: | ||
| failures.append( ('fail', | ||
| "`%s` should be `%s`" % (''.join(current_neg), ''.join(current_pos)), | ||
| url) ) | ||
| except: | ||
| pass | ||
| return failures | ||
|
|
||
| def __init__(self): | ||
| kwargs = { | ||
| 'name': 'fortune', | ||
| 'accept_header': self.accept('html'), | ||
| 'requires_db': True, | ||
| 'args': ['fortune_url'] | ||
| } | ||
| FrameworkTestType.__init__(self, **kwargs) | ||
|
|
||
| def get_url(self): | ||
| return self.fortune_url | ||
|
|
||
| def verify(self, base_url): | ||
| '''Parses the given HTML string and asks the | ||
| FortuneHTMLParser whether the parsed string is a | ||
| valid fortune response | ||
| ''' | ||
|
|
||
| url = base_url + self.fortune_url | ||
| headers, body = self.request_headers_and_body(url) | ||
|
|
||
| _, problems = basic_body_verification(body, is_json_check=False) | ||
|
|
||
| if len(problems) > 0: | ||
| return problems | ||
|
|
||
| parser = FortuneHTMLParser() | ||
| parser.feed(body) | ||
| (valid, diff) = parser.isValidFortune(self.out) | ||
|
|
||
| if valid: | ||
| problems += verify_headers(headers, url, should_be='html') | ||
|
|
||
| if len(problems) == 0: | ||
| return [('pass', '', url)] | ||
| else: | ||
| return problems | ||
| else: | ||
| failures = [] | ||
| failures.append( | ||
| ('fail', 'Invalid according to FortuneHTMLParser', url)) | ||
| failures += self._parseDiffForFailure(diff, failures, url) | ||
| return failures | ||
|
|
||
| def _parseDiffForFailure(self, diff, failures, url): | ||
| '''Example diff: | ||
| --- Valid | ||
| +++ Response | ||
| @@ -1 +1 @@ | ||
| -<!doctype html><html><head><title>Fortunes</title></head><body><table> | ||
| +<!doctype html><html><head><meta></meta><title>Fortunes</title></head><body><div><table> | ||
| @@ -16 +16 @@ | ||
| ''' | ||
|
|
||
| problems = [] | ||
|
|
||
| # Catch exceptions because we are relying on internal code | ||
| try: | ||
| current_neg = [] | ||
| current_pos = [] | ||
| for line in diff[3:]: | ||
| if line[0] == '+': | ||
| current_neg.append(line[1:]) | ||
| elif line[0] == '-': | ||
| current_pos.append(line[1:]) | ||
| elif line[0] == '@': | ||
| problems.append(('fail', | ||
| "`%s` should be `%s`" % ( | ||
| ''.join(current_neg), ''.join(current_pos)), | ||
| url)) | ||
| if len(current_pos) != 0: | ||
| problems.append( | ||
| ('fail', | ||
| "`%s` should be `%s`" % ( | ||
| ''.join(current_neg), ''.join(current_pos)), | ||
| url)) | ||
| except: | ||
| # If there were errors reading the diff, then no diff information | ||
| pass | ||
| return problems |
Oops, something went wrong.