diff --git a/project/test-requirements.txt b/project/test-requirements.txt index 95f521a0..ac767769 100644 --- a/project/test-requirements.txt +++ b/project/test-requirements.txt @@ -8,4 +8,6 @@ Jinja2==2.7.2 autopep8==1.0.2 pytz>2014.2 mock==1.0.1 -Pillow==2.5.1 \ No newline at end of file +Pillow==2.5.1 +factory-boy==2.6.0 +freezegun==0.3.5 diff --git a/project/tests/factories.py b/project/tests/factories.py new file mode 100644 index 00000000..b5005c7a --- /dev/null +++ b/project/tests/factories.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +import factory +import factory.fuzzy + +from silk.models import Request, Response, SQLQuery + + +HTTP_METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'HEAD', 'OPTIONS'] +STATUS_CODES = [200, 201, 300, 301, 302, 401, 403, 404] + + +class SQLQueryFactory(factory.django.DjangoModelFactory): + + query = factory.Sequence(lambda num: u'SELECT foo FROM bar WHERE foo=%s' % num) + traceback = factory.Sequence(lambda num: u'Traceback #%s' % num) + + class Meta: + model = SQLQuery + + +class RequestMinFactory(factory.django.DjangoModelFactory): + + path = factory.Faker('uri_path') + method = factory.fuzzy.FuzzyChoice(HTTP_METHODS) + + class Meta: + model = Request + + +class ResponseFactory(factory.django.DjangoModelFactory): + request = factory.SubFactory(RequestMinFactory) + status_code = factory.fuzzy.FuzzyChoice(STATUS_CODES) + + class Meta: + model = Response diff --git a/project/tests/test_filters.py b/project/tests/test_filters.py index 4d146f24..1ff30ba4 100644 --- a/project/tests/test_filters.py +++ b/project/tests/test_filters.py @@ -51,16 +51,16 @@ def test_view_name_filter(self): requests = [mock_suite.mock_request() for _ in range(0, 10)] r = random.choice(requests) view_name = r.view_name - requuests = models.Request.objects.filter(ViewNameFilter(view_name)) - for r in requuests: + requests = models.Request.objects.filter(ViewNameFilter(view_name)) + for r in requests: self.assertTrue(r.view_name == view_name) def test_path_filter(self): requests = [mock_suite.mock_request() for _ in range(0, 10)] r = random.choice(requests) path = r.path - requuests = models.Request.objects.filter(PathFilter(path)) - for r in requuests: + requests = models.Request.objects.filter(PathFilter(path)) + for r in requests: self.assertTrue(r.path == path) def test_num_queries_filter(self): @@ -99,8 +99,8 @@ def test_time_spent_filter(self): class TestRequestAfterDateFilter(TestCase): def assertFilter(self, dt, f): - requuests = models.Request.objects.filter(f) - for r in requuests: + requests = models.Request.objects.filter(f) + for r in requests: self.assertTrue(r.start_time > dt) @classmethod @@ -128,8 +128,8 @@ def test_after_date_filter_str(self): class TestRequestBeforeDateFilter(TestCase): def assertFilter(self, dt, f): - requuests = models.Request.objects.filter(f) - for r in requuests: + requests = models.Request.objects.filter(f) + for r in requests: self.assertTrue(r.start_time < dt) @classmethod @@ -163,16 +163,16 @@ def test_name_filter(self): profiles = mock_suite.mock_profiles(n=10) p = random.choice(profiles) name = p.name - requuests = models.Profile.objects.filter(NameFilter(name)) - for p in requuests: + requests = models.Profile.objects.filter(NameFilter(name)) + for p in requests: self.assertTrue(p.name == name) def test_function_name_filter(self): profiles = mock_suite.mock_profiles(n=10) p = random.choice(profiles) func_name = p.func_name - requuests = models.Profile.objects.filter(FunctionNameFilter(func_name)) - for p in requuests: + requests = models.Profile.objects.filter(FunctionNameFilter(func_name)) + for p in requests: self.assertTrue(p.func_name == func_name) def test_num_queries_filter(self): @@ -206,4 +206,4 @@ def test_time_spent_filter(self): query_set = time_taken_filter.contribute_to_query_set(query_set) filtered = query_set.filter(time_taken_filter) for f in filtered: - self.assertGreaterEqual(f.time_taken, c) \ No newline at end of file + self.assertGreaterEqual(f.time_taken, c) diff --git a/project/tests/test_models.py b/project/tests/test_models.py new file mode 100644 index 00000000..6012f789 --- /dev/null +++ b/project/tests/test_models.py @@ -0,0 +1,442 @@ +# -*- coding: utf-8 -*- +import datetime +import uuid +import pytz + +from django.test import TestCase +from django.utils import timezone + + +from freezegun import freeze_time + +from silk import models +from .factories import RequestMinFactory, SQLQueryFactory, ResponseFactory + + +# TODO test atomicity + +# http://stackoverflow.com/questions/13397038/uuid-max-character-length +# UUID_MAX_LENGTH = 36 + +# TODO move to separate file test and collection it self +class CaseInsensitiveDictionaryTest(object): + pass + + +class RequestTest(TestCase): + + def setUp(self): + + self.obj = RequestMinFactory.create() + + def test_uuid_is_primary_key(self): + + self.assertIsInstance(self.obj.id, uuid.UUID) + + @freeze_time('2016-01-01 12:00:00') + def test_start_time_field_default(self): + + obj = RequestMinFactory.create() + self.assertEqual(obj.start_time, datetime.datetime(2016, 1, 1, 12, 0, 0, tzinfo=pytz.UTC)) + + def test_total_meta_time_if_have_no_meta_and_queries_time(self): + + self.assertEqual(self.obj.total_meta_time, 0) + + def test_total_meta_time_if_have_meta_time_spent_queries(self): + + obj = RequestMinFactory.create(meta_time_spent_queries=10.5) + self.assertEqual(obj.total_meta_time, 10.5) + + def test_total_meta_time_if_meta_time(self): + + obj = RequestMinFactory.create(meta_time=3.3) + self.assertEqual(obj.total_meta_time, 3.3) + + def test_total_meta_if_self_have_it_meta_and_queries_time(self): + + obj = RequestMinFactory.create(meta_time=3.3, meta_time_spent_queries=10.5) + self.assertEqual(obj.total_meta_time, 13.8) + + def test_time_spent_on_sql_queries_if_has_no_related_SQLQueries(self): + + self.assertEqual(self.obj.time_spent_on_sql_queries, 0) + + # FIXME probably a bug + def test_time_spent_on_sql_queries_if_has_related_SQLQueries_with_no_time_taken(self): + + query = SQLQueryFactory() + self.obj.queries.add(query) + + self.assertEqual(query.time_taken, None) + + with self.assertRaises(TypeError): + self.obj.time_spent_on_sql_queries + + def test_time_spent_on_sql_queries_if_has_related_SQLQueries_and_time_taken(self): + + query1 = SQLQueryFactory(time_taken=3.5) + query2 = SQLQueryFactory(time_taken=1.5) + self.obj.queries.add(query1, query2) + + self.assertEqual(self.obj.time_spent_on_sql_queries, 0) + + def test_time_spent_on_sql_queries_if_has_related_SQLQueries_and_time_taken(self): + + query1 = SQLQueryFactory(time_taken=3.5) + query2 = SQLQueryFactory(time_taken=1.5) + RequestMinFactory().queries.add(query1, query2) + + self.assertEqual(self.obj.time_spent_on_sql_queries, 0) + + def test_headers_if_has_no_encoded_headers(self): + + self.assertIsInstance(self.obj.headers, models.CaseInsensitiveDictionary) + self.assertFalse(self.obj.headers) + + def test_headers_if_has_encoded_headers(self): + + self.obj.encoded_headers = '{"some-header": "some_data"}' + self.assertIsInstance(self.obj.headers, models.CaseInsensitiveDictionary) + self.assertDictEqual(self.obj.headers, {u'some-header': u'some_data'}) + + def test_content_type_if_no_headers(self): + + self.assertEqual(self.obj.content_type, None) + + def test_content_type_if_no_specific_content_type(self): + + self.obj.encoded_headers = '{"foo": "some_data"}' + self.assertEqual(self.obj.content_type, None) + + def test_content_type_if_header_have_content_type(self): + + self.obj.encoded_headers = '{"content-type": "some_data"}' + self.assertEqual(self.obj.content_type, "some_data") + + def test_save_if_have_no_raw_body(self): + + obj = models.Request(path='/some/path/', method='get') + self.assertEqual(obj.raw_body, '') + obj.save() + self.assertEqual(obj.raw_body, '') + + def test_save_if_have_raw_body(self): + + obj = models.Request(path='/some/path/', method='get', raw_body='some text') + obj.save() + self.assertEqual(obj.raw_body, u'some text') + + def test_save_if_have_no_body(self): + + obj = models.Request(path='/some/path/', method='get') + self.assertEqual(obj.body, '') + obj.save() + self.assertEqual(obj.body, '') + + def test_save_if_have_body(self): + + obj = models.Request(path='/some/path/', method='get', body='some text') + obj.save() + self.assertEqual(obj.body, u'some text') + + def test_save_if_have_no_end_time(self): + + obj = models.Request(path='/some/path/', method='get') + self.assertEqual(obj.time_taken, None) + obj.save() + self.assertEqual(obj.time_taken, None) + + @freeze_time('2016-01-01 12:00:00') + def test_save_if_have_end_time(self): + + date = datetime.datetime(2016, 1, 1, 12, 0, 3, tzinfo=pytz.UTC) + obj = models.Request(path='/some/path/', method='get', end_time=date) + obj.save() + self.assertEqual(obj.end_time, date) + self.assertEqual(obj.time_taken, 3000.0) + + +class ResponseTest(TestCase): + + def setUp(self): + + self.obj = ResponseFactory.create() + + def test_uuid_is_primary_key(self): + + self.assertIsInstance(self.obj.id, uuid.UUID) + + def test_is_1to1_related_to_request(self): + + request = RequestMinFactory.create() + resp = models.Response.objects.create(status_code=200, request=request) + + self.assertEqual(request.response, resp) + + def test_headers_if_has_no_encoded_headers(self): + + self.assertIsInstance(self.obj.headers, models.CaseInsensitiveDictionary) + self.assertFalse(self.obj.headers) + + def test_headers_if_has_encoded_headers(self): + + self.obj.encoded_headers = '{"some-header": "some_data"}' + self.assertIsInstance(self.obj.headers, models.CaseInsensitiveDictionary) + self.assertDictEqual(self.obj.headers, {u'some-header': u'some_data'}) + + def test_content_type_if_no_headers(self): + + self.assertEqual(self.obj.content_type, None) + + def test_content_type_if_no_specific_content_type(self): + + self.obj.encoded_headers = '{"foo": "some_data"}' + self.assertEqual(self.obj.content_type, None) + + def test_content_type_if_header_have_content_type(self): + + self.obj.encoded_headers = '{"content-type": "some_data"}' + self.assertEqual(self.obj.content_type, "some_data") + + +class SQLQueryManagerTest(TestCase): + + def test_if_no_args_passed(self): + pass + def test_if_one_arg_passed(self): + pass + def if_a_few_args_passed(self): + pass + def if_objs_kw_arg_passed(self): + pass + def if_not_the_objs_kw_arg_passed(self): + pass + + +class SQLQueryTest(TestCase): + + def setUp(self): + + self.obj = SQLQueryFactory.create() + self.end_time = datetime.datetime(2016, 1, 1, 12, 0, 5, tzinfo=pytz.UTC) + self.start_time = datetime.datetime(2016, 1, 1, 12, 0, 0, tzinfo=pytz.UTC) + + @freeze_time('2016-01-01 12:00:00') + def test_start_time_field_default(self): + + obj = SQLQueryFactory.create() + self.assertEqual(obj.start_time, datetime.datetime(2016, 1, 1, 12, 0, 0, tzinfo=pytz.UTC)) + + def test_is_m2o_related_to_request(self): + + request = RequestMinFactory() + self.obj.request = request + self.obj.save() + + self.assertIn(self.obj, request.queries.all()) + + def test_query_manager_instance(self): + + self.assertIsInstance(models.SQLQuery.objects, models.SQLQueryManager) + + def test_traceback_ln_only(self): + + self.obj.traceback = """Traceback (most recent call last): + File "/home/user/some_script.py", line 10, in some_func + pass + File "/usr/lib/python2.7/bdb.py", line 20, in trace_dispatch + return self.dispatch_return(frame, arg) + File "/usr/lib/python2.7/bdb.py", line 30, in dispatch_return + if self.quitting: raise BdbQuit + BdbQuit""" + + output = ('Traceback (most recent call last):\n' + ' pass\n' + ' return self.dispatch_return(frame, arg)\n' + ' if self.quitting: raise BdbQuit') + + self.assertEqual(self.obj.traceback_ln_only, output) + + def test_formatted_query_if_no_query(self): + + self.obj.query = "" + self.obj.formatted_query + + def test_formatted_query_if_has_a_query(self): + + query = """SELECT Book.title AS Title, + COUNT(*) AS Authors + FROM Book + JOIN Book_author + ON Book.isbn = Book_author.isbn + GROUP BY Book.title;""" + + self.obj.query = query + self.obj.formatted_query + + def test_num_joins_if_no_joins_in_query(self): + + query = """SELECT Book.title AS Title, + COUNT(*) AS Authors + FROM Book + GROUP BY Book.title;""" + + self.obj.query = query + + self.assertEqual(self.obj.num_joins, 0) + + def test_num_joins_if_joins_in_query(self): + + query = """SELECT p.id + FROM Person p + JOIN address a ON p.Id = a.Person_ID + JOIN address_type at ON a.Type_ID = at.Id + JOIN `option` o ON p.Id = o.person_Id + JOIN option_address_type oat ON o.id = oat.option_id + WHERE a.country_id = 1 AND at.id <> oat.type_id;""" + + self.obj.query = query + self.assertEqual(self.obj.num_joins, 4) + + # FIXME bug, not a feature + def test_num_joins_if_no_joins_in_query_but_this_word_searched(self): + + query = """SELECT Book.title FROM Book WHERE Book.title=`Join the dark side, Luke!`;""" + + self.obj.query = query + self.assertEqual(self.obj.num_joins, 1) + + def test_tables_involved_if_no_query(self): + + self.obj.query = '' + + self.assertEqual(self.obj.tables_involved, []) + + def test_tables_involved_if_query_has_only_a_from_token(self): + + query = """SELECT * FROM Book;""" + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['Book;']) + + def test_tables_involved_if_query_has_a_join_token(self): + + query = """SELECT p.id FROM Person p JOIN Address a ON p.Id = a.Person_ID;""" + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['Person', 'Address']) + + def test_tables_involved_if_query_has_an_as_token(self): + + query = 'SELECT Book.title AS Title FROM Book GROUP BY Book.title;' + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['Title', 'Book']) + + # FIXME bug, not a feature + def test_tables_involved_check_with_fake_a_from_token(self): + + query = """SELECT * FROM Book WHERE Book.title=`EVIL FROM WITHIN`;""" + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['Book', 'WITHIN`;']) + + # FIXME bug, not a feature + def test_tables_involved_check_with_fake_a_join_token(self): + + query = """SELECT * FROM Book WHERE Book.title=`Luke, join the dark side!`;""" + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['Book', 'the']) + + # FIXME bug, not a feature + def test_tables_involved_check_with_fake_an_as_token(self): + + query = """SELECT * FROM Book WHERE Book.title=`AS SOON AS POSIABLE`;""" + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['Book', 'POSIABLE`;']) + + def test_tables_involved_if_query_has_subquery(self): + + query = '''SELECT A.Col1, A.Col2, B.Col1,B.Col2 + FROM (SELECT RealTableZ.Col1, RealTableY.Col2, RealTableY.ID AS ID + FROM RealTableZ + LEFT OUTER JOIN RealTableY ON RealTableZ.ForeignKeyY=RealTableY.ID + WHERE RealTableY.Col11>14 + ) AS B INNER JOIN A + ON A.ForeignKeyY=B.ID;''' + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['ID', 'RealTableZ', 'RealTableY', 'B', 'A']) + + # FIXME bug, not a feature + def test_tables_involved_if_query_has_django_aliase_on_column_names(self): + + query = 'SELECT foo AS bar FROM some_table;' + self.obj.query = query + self.assertEqual(self.obj.tables_involved, ['bar', 'some_table;']) + + def test_save_if_no_end_and_start_time(self): + + obj = SQLQueryFactory.create() + + self.assertEqual(obj.time_taken, None) + + @freeze_time('2016-01-01 12:00:00') + def test_save_if_has_end_time(self): + + # datetime.datetime(2016, 1, 1, 12, 0, 5, tzinfo=pytz.UTC) + obj = SQLQueryFactory.create(end_time=self.end_time) + + self.assertEqual(obj.time_taken, 5000.0) + + @freeze_time('2016-01-01 12:00:00') + def test_save_if_has_start_time(self): + + obj = SQLQueryFactory.create(start_time=self.start_time) + + self.assertEqual(obj.time_taken, None) + + def test_save_if_has_end_and_start_time(self): + + obj = SQLQueryFactory.create(start_time=self.start_time, end_time=self.end_time) + + self.assertEqual(obj.time_taken, 5000.0) + + def test_save_if_has_pk_and_request(self): + + self.obj.request = RequestMinFactory.create() + self.obj.save() + self.assertEqual(self.obj.request.num_sql_queries, 0) + + def test_save_if_has_no_pk(self): + + obj = SQLQueryFactory.build(start_time=self.start_time, end_time=self.end_time) + obj.request = RequestMinFactory.create() + obj.save() + self.assertEqual(obj.request.num_sql_queries, 1) + + # should not rise + def test_save_if_has_no_request(self): + + obj = SQLQueryFactory.build(start_time=self.start_time, end_time=self.end_time) + obj.save() + + # FIXME a bug + def test_delete_if_no_related_requests(self): + + with self.assertRaises(AttributeError): + self.obj.delete() + + # self.assertNotIn(self.obj, models.SQLQuery.objects.all()) + + def test_delete_if_has_request(self): + + self.obj.request = RequestMinFactory.create() + self.obj.save() + self.obj.delete() + + self.assertNotIn(self.obj, models.SQLQuery.objects.all()) + + +class BaseProfileTest(TestCase): + pass + + +class ProfileTest(TestCase): + pass diff --git a/silk/models.py b/silk/models.py index 63df749f..9cb257a0 100644 --- a/silk/models.py +++ b/silk/models.py @@ -53,7 +53,7 @@ class Request(models.Model): view_name = CharField(max_length=300, db_index=True, blank=True, default='', null=True) end_time = DateTimeField(null=True, blank=True) time_taken = FloatField(blank=True, null=True) - encoded_headers = TextField(blank=True, default='') + encoded_headers = TextField(blank=True, default='') # stores json meta_time = FloatField(null=True, blank=True) meta_num_queries = IntegerField(null=True, blank=True) meta_time_spent_queries = FloatField(null=True, blank=True) @@ -66,7 +66,7 @@ def total_meta_time(self): # defined in atomic transaction within SQLQuery save()/delete() as well # as in bulk_create of SQLQueryManager # TODO: This is probably a bad way to do this, .count() will prob do? - num_sql_queries = IntegerField(default=0) + num_sql_queries = IntegerField(default=0) # TODO replace with count() @property def time_spent_on_sql_queries(self): @@ -84,6 +84,7 @@ def headers(self): raw = json.loads(self.encoded_headers) else: raw = {} + return CaseInsensitiveDictionary(raw) @property @@ -94,12 +95,14 @@ def save(self, *args, **kwargs): # sometimes django requests return the body as 'None' if self.raw_body is None: self.raw_body = '' + if self.body is None: self.body = '' if self.end_time and self.start_time: interval = self.end_time - self.start_time self.time_taken = interval.total_seconds() * 1000 + super(Request, self).save(*args, **kwargs) @@ -124,6 +127,7 @@ def headers(self): return CaseInsensitiveDictionary(raw) +# TODO rewrite docstring class SQLQueryManager(models.Manager): def bulk_create(self, *args, **kwargs): """ensure that num_sql_queries remains consistent. Bulk create does not call @@ -132,6 +136,7 @@ def bulk_create(self, *args, **kwargs): objs = args[0] else: objs = kwargs.get('objs') + with atomic(): request_counter = Counter([x.request_id for x in objs]) requests = Request.objects.filter(pk__in=request_counter.keys()) @@ -155,6 +160,7 @@ class SQLQuery(models.Model): traceback = TextField() objects = SQLQueryManager() + # TODO docstring @property def traceback_ln_only(self): return '\n'.join(self.traceback.split('\n')[::2]) @@ -174,13 +180,15 @@ def tables_involved(self): TODO: Can probably parse the SQL using sqlparse etc and pull out table info that way?""" components = [x.strip() for x in self.query.split()] tables = [] - for idx, c in enumerate(components): + + for idx, component in enumerate(components): # TODO: If django uses aliases on column names they will be falsely identified as tables... - if c.lower() == 'from' or c.lower() == 'join' or c.lower() == 'as': + if component.lower() == 'from' or component.lower() == 'join' or component.lower() == 'as': try: - nxt = components[idx + 1] - if not nxt.startswith('('): # Subquery - stripped = nxt.strip().strip(',') + _next = components[idx + 1] + if not _next.startswith('('): # Subquery + stripped = _next.strip().strip(',') + if stripped: tables.append(stripped) except IndexError: # Reach the end @@ -189,13 +197,16 @@ def tables_involved(self): @atomic() def save(self, *args, **kwargs): + if self.end_time and self.start_time: interval = self.end_time - self.start_time self.time_taken = interval.total_seconds() * 1000 + if not self.pk: if self.request: self.request.num_sql_queries += 1 self.request.save() + super(SQLQuery, self).save(*args, **kwargs) @atomic()