diff --git a/tempesta_fw/t/functional/README.md b/tempesta_fw/t/functional/README.md new file mode 100644 index 000000000..91c4f5da4 --- /dev/null +++ b/tempesta_fw/t/functional/README.md @@ -0,0 +1,56 @@ + +# Tempesta FW Functional tests. + +## Prerequisites + +Before runing functional tests you need to compile the Tempesta with its prerequisites. +Most of the functional test use the Apache web server as back-end. So you need to install it. +A Integrated test back-end requires package python-setproctitle. + + +## Run tests + +To run one test you can use from a tempesta directory: + + $ tempesta_fw/t/functional/run_tests.sh [test_name] + +Or: + + $ tempesta_fw/t/functional/tests.py [test_name] + +To run all tests: + + $ tempesta_fw/t/functional/run_tests.sh all + +Or: + + $ tempesta_fw/t/functional/tests.py all + +To set the ip port of the test http backend server you can use a parameter: + +-p . + +For example: + + $ tempesta_fw/t/functional/run_tests.sh test_parser -p 8080 + + + + +## Add new tests + +To add new tests you need to write a new functional tests in python and +put it in directory: + + $ tempesta_fw/t/functional/tests/ + +And add the name of the new test in the init-file: + + + $ tempesta_fw/t/functional/tests/\_\_init\_\_.py. + +When you write functional test, you can use helper modules from + + $ tempesta_fw/t/functional/test/helpers/ + + diff --git a/tempesta_fw/t/functional/fragmented_requests.py b/tempesta_fw/t/functional/fragmented_requests.py deleted file mode 100755 index a5118075f..000000000 --- a/tempesta_fw/t/functional/fragmented_requests.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 - -from helpers import * - -__author__ = 'NatSys Lab' -__copyright__ = 'Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).' -__license__ = 'GPL2' - -req_get = '''\ -GET http://github.com/natsys/tempesta HTTP/1.1\r -Host: github.com\r -User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0 Iceweasel/31.2.0\r -Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r -Accept-Language: en-US,en;q=0.5\r -Accept-Encoding: gzip, deflate\r -DNT: 1\r -Referer: http://natsys-lab.com/cgi-bin/show.pl\r -Cookie: logged_in=yes; _ga=GA1.2.1404175546.1417001200; user_session=EdI8qD-H305ePHXkP13VfCIDNAKgSSxdEGq25wtENSwxsxRKJVIDstdZLU_9EYy68Dj7jBKVtF9G9Kxel; dotcom_user=vdmit11; _octo=GH1.1.1046670168.1410702951; _gh_sess=eyJzZXNba9WuX2lkIjoiMDY5ZmM5MGFmMTFjZDgxZTIxNzY0MTNlM2M3YzBmmMIiLCJzcHlfcmVwbyI6Im5hdHN5cy90ZW1ZwXN0YSIsInNweV9yZXBvX2F0IjoxNDE3NzM1MzQ5LCJjb250ZXh0IjoiLyIsImxhc3Rfd3JpdGUijOE9MTc3MzUzNDk3NDN7--eed6d44a1be9e83a34dbf8d5e319a520f30fa481; tz=Europe%2FMoscow; _gat=1\r -Connection: Keep-Alive\r -Cache-Control: max-age=0\r - -''' - -def validate_received_req_get(method, path, headers, body): - assert method == 'GET' - assert path == 'http://github.com/natsys/tempesta' - assert len(body) == 0 - h = { - 'Host': 'github.com', - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0 Iceweasel/31.2.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - 'DNT': '1', - 'Referer': 'http://natsys-lab.com/cgi-bin/show.pl', - 'Cookie': 'logged_in=yes; _ga=GA1.2.1404175546.1417001200; user_session=EdI8qD-H305ePHXkP13VfCIDNAKgSSxdEGq25wtENSwxsxRKJVIDstdZLU_9EYy68Dj7jBKVtF9G9Kxel; dotcom_user=vdmit11; _octo=GH1.1.1046670168.1410702951; _gh_sess=eyJzZXNba9WuX2lkIjoiMDY5ZmM5MGFmMTFjZDgxZTIxNzY0MTNlM2M3YzBmmMIiLCJzcHlfcmVwbyI6Im5hdHN5cy90ZW1ZwXN0YSIsInNweV9yZXBvX2F0IjoxNDE3NzM1MzQ5LCJjb250ZXh0IjoiLyIsImxhc3Rfd3JpdGUijOE9MTc3MzUzNDk3NDN7--eed6d44a1be9e83a34dbf8d5e319a520f30fa481; tz=Europe%2FMoscow; _gat=1', - 'Connection': 'Keep-Alive', - 'Cache-Control': 'max-age=0', - } - for k in h.keys(): - if h[k] != headers[k]: - msg = "Expected: %s, got: %s" % (h[k], headers[k]) - raise Error(msg) - - -def fragmentize_str(s, frag_size): - """ - Split a string into a list of equal N-sized fragments. - - >>> fragmentize_str("foo12bar34baz", 3) - ['foo', '12b', 'ar3', '4ba', 'z'] - """ - return [ s[i:i+frag_size] for i in range(0, len(s), frag_size) ] - - -backend_callback_counter = 0 - -def backend_callback(method, path, headers, body): - ++backend_callback_counter; - validate_received_req_get(method, path, headers, body) - return 200, { 'Content-Type': 'text/plan' }, 'Everything is OK.' - - -# Start Tempesta FW and a back-end server with a default configuration. -tfw.start() -be.start(backend_callback) - - -# The test body: -# -# In the real world, HTTP messages, especially big ones, are often fragmented -# (because of network MTU/MRU, TCP MSS, application buffer sizes, etc). -# Fragments of the same HTTP request may have different sizes and arrive to -# server at different time. -# -# The goal here is to simulate such situation and check that the parser of -# Tempesta FW can handle messages broken to any number of such fragments. -# -# We would like to go the most extreme case: split the request to 1-byte chunks -# to check that a gap may occur at any position of the message. -# But Tempesta FW assumes (for optimization purposes) that first few bytes are -# continuous (and indeed in the real world they are), so we put them as a solid -# fragment, and split the rest of the message to single characters. -# -# So in the end we should get the following fragments sent to the server: -# [ 'GET http://', 'g', 'i', 't', 'h', 'u', 'b', '.', 'c', 'o', 'm', ... ] - - -with cli.connect_to_tfw() as socket: - socket.sendall(bytes(req_get, 'UTF-8')) - # wait for a response from Tempesta, just receive anything, we don't care what - socket.recv(1) - assert backend_callback_counter == 1 - - - diff --git a/tempesta_fw/t/functional/helpers/__init__.py b/tempesta_fw/t/functional/helpers/__init__.py deleted file mode 100644 index f758b7ceb..000000000 --- a/tempesta_fw/t/functional/helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = [ 'be', 'cli', 'tfw' ] diff --git a/tempesta_fw/t/functional/helpers/be.py b/tempesta_fw/t/functional/helpers/be.py deleted file mode 100644 index a8df0f164..000000000 --- a/tempesta_fw/t/functional/helpers/be.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -A primitive back-end HTTP server implementation suitable for testing purposes. -""" - -from http.server import * -from threading import * - -__author__ = 'NatSys Lab' -__copyright__ = 'Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).' -__license__ = 'GPL2' - -def start(*args, **kwargs): - """A shortcut for BackendHTTPServer() constructor.""" - return BackendHTTPServer(*args, **kwargs) - -def _dummy_callback(method, uri, headers, body): - """An example of a backend_callback passed to BackendHTTPServer().""" - ret_code = 200 - ret_headers = { 'Content-Type': 'text/html; charset=utf-8' } - ret_body = 'Hello from dummy back-end callback' - return (ret_code, ret_headers, ret_body) - -class BackendHTTPServer(Thread, HTTPServer): - """ - Basically, this implementation does two things: - 1. It runs in a HTTP server in a separate thread. - 2. It handles all HTTP requests with a single backend_callback function - passed to the constructor. - - Also, right after initialization it blocks until a first TCP connection is - accepted. That is done to wait until Tempesta FW is connected. - So you have to start Tempesta first, and only then spawn the HTTP server. - """ - def __init__(self, backend_callback=_dummy_callback, port=8080, - wait_tfw_timeout_sec=20): - # Initialize HTTP server, bind/listen/etc. - self.accept_event = Event() - self.backend_callback = backend_callback - HTTPServer.__init__(self, ('127.0.0.1', port), BackendHTTPRequestHandler) - - # Start a background thread that accept()s connections. - kwargs = dict(poll_interval = 0.05) - Thread.__init__(self, target=self.serve_forever, kwargs=kwargs) - self.start() - - # Synchronize with Tempesta FW. - if (wait_tfw_timeout_sec): - self.wait_for_tfw(wait_tfw_timeout_sec) - - def wait_for_tfw(self, timeout): - """ - Sleep until a first accepted connection (presumably from Tempesta FW). - - FIXME: race conditions possible: after the connection is established, - the Tempesta FW must add the server to a load-balancing scheduler and - so on, so there is a time span, when the Tempesta is not yet ready to - forward incoming requests to the connected back-end server. At this - point we just hope that this time span is negligible. - BTW, that may be fixed by exporting state of Temepsta FW via debugfs. - """ - got_connection = self.accept_event.wait(timeout) - if (not got_connection): - self.shutdown() - msg = ("No connection from Tempesta FW (backend: {0}, timeout: {1})" - .format(self.server_address, timeout)) - raise Exception(msg) - - # get_request() calls accept() that blocks until the first connection. - # We just inject a synchronization with wait_for_tfw() there. - def get_request(self): - ret = super().get_request() - self.accept_event.set() - return ret - -class BackendHTTPRequestHandler(BaseHTTPRequestHandler): - """ - A wrapper for BackendHTTPServer.backend_callback. - The class simply pushes HTTP requests to the callback, and then builds - responses from data returned by the callback. - - That is done for simplicity. It is easier to code a single callback function - than a whole handler class. We have to code one in every test, and we don't - need much power in tests code, so we prefer a function over a class. - """ - def _handle_req_with_cb(self): - """ - Pass HTTP request to backend_callback, send a response containing data - returned by the callback. - """ - # Read body and push it to the callback - headers = self.headers - body_len = int(headers['Content-Length'] or 0) - body = self.rfile.read(body_len) - cb = self.server.backend_callback - resp_tuple = cb(self.command, self.path, headers, body) - - # The callback must return a tuple of exactly 3 elements: - # (http_code, headers_dict, body_str) - assert len(resp_tuple) == 3 - - # Send response fields provided by the callback. - code, headers, body = resp_tuple - body = bytes(body, 'UTF-8') - self.send_response(code) - for name, val in headers.items(): - self.send_header(name, val) - self.end_headers() - self.wfile.write(body) - print(body) - - # At this point Tempesta FW parser blocks HTTP/1.0 requests - protocol_version = 'HTTP/1.1' - - # Actual handler methods. We dispatch all them into our single function. - do_GET = _handle_req_with_cb - do_POST = _handle_req_with_cb - # Add do_METHOD here if you need anything beyond GET and POST methods. - - # By default, the base class prints a message for every incoming request. - # We don't want to see this flood in test results, so here is the stub. - def log_message(self, format, *args): - return diff --git a/tempesta_fw/t/functional/helpers/cli.py b/tempesta_fw/t/functional/helpers/cli.py deleted file mode 100644 index c7e164aee..000000000 --- a/tempesta_fw/t/functional/helpers/cli.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -HTTP client emulator. - -These tests are built around a network of three participants: client, server and -a running Tempesta FW instance. This module is responsible for the client part. -It allows to connect to the Tempesta and send some data to it in various ways. -""" - -from socket import * -from contextlib import contextmanager - -__author__ = 'NatSys Lab' -__copyright__ = 'Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).' -__license__ = 'GPL2' - -@contextmanager -def connect_to_tfw(port=80, timeout_sec=5): - socket = create_connection(('127.0.0.1', port), timeout_sec) - yield socket - socket.close() diff --git a/tempesta_fw/t/functional/helpers/teardown.py b/tempesta_fw/t/functional/helpers/teardown.py deleted file mode 100644 index d6ed47a1f..000000000 --- a/tempesta_fw/t/functional/helpers/teardown.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Implementation of a "teardown" routine which is executed after every test. - -A problem: Python interpreter doesn't exit until all threads are exited. -That means that when a test finishes, it hangs if there are backend threads -running in background. So these threads should be terminated somehow. -We don't want to write some boilerplate code at the end of each test, -so here we have code that does it automatically. -""" - -import os -import sys -from threading import * - -__author__ = 'NatSys Lab' -__copyright__ = 'Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).' -__license__ = 'GPL2' - -_main_thread = current_thread() -_teardown_hooks = [] -_stderr_write_event = Event() - -def register(hook_fn, *args, **kwargs): - """ - Register a hook which is executed when the main thread terminates. - - The atexit.register() doesn't work because we terminate all threads - asynchronously via os._exit(). So this function is a drop-in replacement - that works in our case. - """ - entry = (hook_fn, args, kwargs) - _teardown_hooks.append(entry) - -def _call_hooks_when_main_exits(): - """ - Wait until the main thread exits, and then terminate all remaining threads - by sending them a signal. Also execute all register()'ed hooks. - """ - _main_thread.join() - for hook_fn, args, kwargs in _teardown_hooks: - hook_fn(*args, **kwargs) - os._exit(-1 if _stderr_write_event.is_set() else 0) - -# Start another thread to catch a moment when the main thread exits. -# It turns out there is no cleaner solution because the Thread class doesn't -# provide any way to add exit hooks. -Thread(target=_call_hooks_when_main_exits).start() - -# Unfortunately, we can't obtain an exit code of a Thread after it exits. -# So we detect errors by the fact of writing to the stderr stream. -# We wrap sys.stderr and set the error flag if someone writes a message there. -class StdstreamWrapper(): - def __init__(self, stream): - self.stream = stream - - def write(self, data): - _stderr_write_event.set() - self.stream.write(data) - -sys.stderr = StdstreamWrapper(sys.stderr) diff --git a/tempesta_fw/t/functional/helpers/tfw.py b/tempesta_fw/t/functional/helpers/tfw.py deleted file mode 100644 index 8beeffa9e..000000000 --- a/tempesta_fw/t/functional/helpers/tfw.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -""" -Helpers for interacting with Tempesta FW (start/stop, configure, etc). -""" - -import os -import subprocess -import sys - -from . import teardown - -__author__ = 'NatSys Lab' -__copyright__ = 'Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).' -__license__ = 'GPL2' - -_functest_dir = os.path.dirname(os.path.realpath(sys.argv[0])) -_tempesta_dir = os.path.normpath(os.path.join(_functest_dir, '../../../')) - -def start(): - _sh("SYNC_SOCKET=./sync_socket TDB=./tempesta_db ./tempesta.sh start") - -def stop(): - _sh("./tempesta.sh stop") - -def _sh(command): - return subprocess.check_output(command, shell=True, cwd=_tempesta_dir) - -def _is_started(): - return (0 == subprocess.call("lsmod | grep -q tempesta", shell=True)) - -def _stop_if_started(): - if (_is_started()): - stop() - -# Ensure we start and stop in a pristine environment. -assert (not _is_started()) -# The teardown line is commented-out because we have the issue: -# #10 -Oops on shutdown -# At this point it is not solved and Tempesta FW simply can't be stopped. -# TODO: un-comment it after the issue is fixed. -#teardown.register(_stop_if_started) diff --git a/tempesta_fw/t/functional/run_all_tests.sh b/tempesta_fw/t/functional/run_all_tests.sh deleted file mode 100755 index 6e3d43f73..000000000 --- a/tempesta_fw/t/functional/run_all_tests.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# -# 2014. Written by NatSys Lab. (info@natsys-lab.com). - -function run() { - echo run: $1 - $(dirname $0)/$1 - if [ $? -ne 0 ] - then - echo FAILED: $1 - exit -1 - fi - echo PASSED: $1 -} - -echo -echo ------------------------------------------------------------------ -echo Running functional tests... -echo ------------------------------------------------------------------ - -# Doesn't pass yet. -# run fragmented_requests.py diff --git a/tempesta_fw/t/functional/run_tests.sh b/tempesta_fw/t/functional/run_tests.sh new file mode 100644 index 000000000..ee7a7635f --- /dev/null +++ b/tempesta_fw/t/functional/run_tests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright(C) 2014 Natsys Lab. (info@natsys-lab.com) + #Copyright(C) 2015 - 2017. Tempesta Technologies Inc. + +function run() { + echo run: $1 + $(dirname $0)/$1 + if [ $? -ne 0 ] + then + echo FAILED: $1 + exit -1 + fi +} +if [$1 == ""]; then +echo -e "Bad parameters:\nusage:\nrun_tests.sh \nor run_tests.sh all\n" +else +$(dirname $0)/tests.py $@ +#echo +#echo ------------------------------------------------------------------ +#echo Running functional tests... +#echo ------------------------------------------------------------------ + +#run tests.py $1 +fi diff --git a/tempesta_fw/t/functional/tests.py b/tempesta_fw/t/functional/tests.py new file mode 100755 index 000000000..df1ed827f --- /dev/null +++ b/tempesta_fw/t/functional/tests.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016-2017 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +# It runs tests from +# $ tempesta_fw/t/functional/tests. +# The names of the tests have to be in the __init__.py file in that directory. + +import pkgutil +import tests + +import subprocess +import sys +import os +from os.path import dirname, realpath, sep + +sys.path.append((dirname(realpath(__file__))+ sep + "tests" + sep + "helpers")) +"""For tests.py the root is on three levels up.""" +dir = dirname(realpath(__file__)) +dir = dirname(dir) +dir = dirname(dir) +root = dirname(dir) +import conf +import teardown +conf.set_root(root) + +if len(sys.argv) > 1: + argindex = 0 + for arg in sys.argv: + if arg == '-p': + conf.set_beport(int(sys.argv[argindex + 1])) + argindex += 1 + for loader, name, ispkg in pkgutil.iter_modules(path = tests.__path__, + prefix = ''): + if not ispkg: + if (name == sys.argv[1]) | (sys.argv[1] == "all"): + test = loader.find_module(name).load_module(name) + tclass = getattr(test, 'Test') + print("test:{}".format(tclass().get_name())) + tclass().run() +else: + print("\n\nusage:\n test.py or test.py all\n") + os._exit(0) + + diff --git a/tempesta_fw/t/functional/tests/__init__.py b/tempesta_fw/t/functional/tests/__init__.py new file mode 100644 index 000000000..cf07ad726 --- /dev/null +++ b/tempesta_fw/t/functional/tests/__init__.py @@ -0,0 +1,6 @@ +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016-2017 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +__all__ = [ "bomber", "fragmented_requests", "test_frang", "test_cache", + "test_parser" ] diff --git a/tempesta_fw/t/functional/tests/bomber.py b/tempesta_fw/t/functional/tests/bomber.py new file mode 100644 index 000000000..7171719dc --- /dev/null +++ b/tempesta_fw/t/functional/tests/bomber.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# A test just run bomber script on the Tempesta. +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 20115-2017 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +import conf +import tfw + +c = conf.TFWConfig() + +class Test: + def get_name(self): + return 'bomber' + + def run(self): + """ The functions sets a simple configuration and starts + the Tempesta. Then it starts the bomber script. If the bomber + runs without errors, then this simple test is passed. + """ + c.add_option('cache', '0') + c.add_option('listen', '8081') + c.add_option('server', '127.0.0.1:80') + tfw.start() + print("tfw started\n") + tfw.start_bomber() + print("bomber started\n") + tfw.stop() + print("tfw stoped\n") + diff --git a/tempesta_fw/t/functional/tests/fragmented_requests.py b/tempesta_fw/t/functional/tests/fragmented_requests.py new file mode 100644 index 000000000..d9ac01a6d --- /dev/null +++ b/tempesta_fw/t/functional/tests/fragmented_requests.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +#A test for the Tempesta http parser with a request divided into chunks. +# We get a request, divide it into chunks, then send the chunks separately. +# Then check the Tempesta response. + +__author__ = 'Temesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +import conf +import tfw +import socket +import tfwparser +import be +import datetime +class Test: + def fragmentize_str(self, s, frag_size): + """ + Split a string into a list of equal N-sized fragmen. + fragmentize_str("foo12bar34baz", 3) + ['foo', '12b', 'ar3', '4ba', 'z'] + """ + return [s[i:i+frag_size] for i in range(0, len(s), frag_size)] + + def run(self): + req_get = 'GET http://github.com/natsys/tempesta HTTP/1.1\r\n' + req_get += 'Host: github.com\r\n' + req_get += 'User-Agent: Mozilla/5.0 (X11; Linux x86_64;' + req_get += ' rv:31.0) Gecko/20100101 Firefox/31.0' + req_get += 'Iceweasel/31.2.0\r\n' + req_get += 'Accept: text/html,application/xhtml+xml,' + req_get += 'application/xml;q=0.9,*/*;q=0.8\r\n' + req_get += 'Accept-Language: en-US,en;q=0.5\r\n' + req_get += 'Accept-Encoding: gzip, deflate\r\n' + req_get += 'Referer: http://natsys-lab.com/cgi-bin/show.pl' + req_get += '\r\n\r\n' + c = conf.TFWConfig() + parser = tfwparser.TFWParser() + c.add_option('cache', '0') + c.add_option('listen', '8081') + c.add_option('server', '127.0.0.1:80') + body_md5 = "" + tfw.start() + print("tfw start\n") + self.res = True + + fragment_size =1 + while fragment_size < len(req_get): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect(("127.0.0.1",8081)) + s.settimeout(1) + for fs in self.fragmentize_str(req_get, fragment_size): + s.sendall(fs) + data = s.recv(1024) + if len(data) == 0: + self.res = False + else: + status = parser.get_status(data) + if body_md5 == "": + body_md5 = parser.get_body_hash(data) + else: + if parser.get_body_hash(data) !=\ + body_md5: + print("body_hash not match \n") + self.res = False + if (status != 200) & (status != 404): + print("status:{}\n".format(status)) + self.res = False + s.close() + fragment_size += 1 + + s.close() + tfw.stop() + print("Res:{}\n".format(self.res)) + + def get_name(self): + return 'fragmented request' + diff --git a/tempesta_fw/t/functional/tests/helpers/__init__.py b/tempesta_fw/t/functional/tests/helpers/__init__.py new file mode 100644 index 000000000..d8d00ffe0 --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/__init__.py @@ -0,0 +1,5 @@ +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies. (info@natsys-lab.com).' +__license__ = 'GPL2' + +__all__ = [ 'be', 'cli', 'tfw','conf', 'apache', 'tfwparser' ] diff --git a/tempesta_fw/t/functional/tests/helpers/apache.py b/tempesta_fw/t/functional/tests/helpers/apache.py new file mode 100644 index 000000000..4742de148 --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/apache.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies. (info@natsys-lab.com).' +__license__ = 'GPL2' + +import subprocess + + +def is_apache(): + try: + dist = get_dist() + if dist == "debian": + p = subprocess.Popen(["apache2", "-v"], + stdout=subprocess.PIPE) + elif dist == "centos": + p = subprocess.Popen(["httpd", "-v"], + stdout=subprocess.PIPE) + except OSError as e: + if e.errno == 2: + return False + out = p.stdout.read() + print("out:{}".format(out)) + if out.find("not found") > 0: + return False + else: + return True + +def get_dist(): + p = subprocess.Popen(["cat", "/etc/os-release"], stdout=subprocess.PIPE) + out = p.stdout.read() + if out.find("ID=debian") > 0: + return "debian" + else: + if out.find("ID=\"centos\"") > 0: + return "centos" + +def start(): + distr = get_dist() + if is_apache(): + if distr == "debian": + subprocess.call("service apache2 start", shell = True) + elif distr == "centos": + subprocess.call("service httpd start", shell = True) + + else: + print("apache is not installed\n") diff --git a/tempesta_fw/t/functional/tests/helpers/be.py b/tempesta_fw/t/functional/tests/helpers/be.py new file mode 100644 index 000000000..1350b7af2 --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/be.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +# A primitive back-end HTTP server implementation suitable for testing purposes. + +import os +import sys +import signal +import SimpleHTTPServer +import BaseHTTPServer +from SimpleHTTPServer import SimpleHTTPRequestHandler +import socket +import datetime +import time +import setproctitle +import conf + +server = None + +def handler(signum, frame): + os.remove('be.pid') + os._exit(0) + +def start(unlim, resp): + be_port = conf.get_beport() + httpd = Server(('127.0.0.1', be_port), BackendHTTPRequestHandler) + httpd.set_unlim(unlim) + httpd.set_resp(resp) + server = httpd + server.socket.setblocking(0) + pid = os.fork() + if pid == 0: + os.setsid() + pid = os.fork() + if pid == 0: + setproctitle.setproctitle("tfwbackend") + wd = os.open("be.pid", os.O_RDWR | os.O_CREAT) + w = os.fdopen(wd, 'w') + s_pid = str(os.getpid()) + w.write(s_pid) + w.flush() + w.close() + signal.signal(signal.SIGUSR1, handler) + server.serve_forever() + else: + os._exit(0) + """Wait while the child writes his pid. We can't wait it to + the end (serve forever). So let it be a fraction of a second. + """ + time.sleep(0.2) + rd = open("be.pid", 'a+') + r_pid = rd.read() + rd.close() + return int(r_pid) + +def stop(pid): + if server != None: + server.server_close() + server.shutdown() + server.keep = False + os.kill(pid, signal.SIGUSR1) + +class Server(BaseHTTPServer.HTTPServer): + def set_unlim(self,unlim): + self.unlim = unlim + + def set_resp(self, resp): + self.resp = resp + +class BackendHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def __init__(self, req, client_address, server): + + self. protocol_version = b'HTTP/1.1' + + SimpleHTTPRequestHandler.__init__(self, req, client_address, + server) + + def log_error(self, fmt, args): +# Supress a no essential timeout message. + pass + + def handle(self): + self.resp = self.server.resp + if self.server.unlim == False: + self.close_connection = 0 + else: + self.close_connection = 1 + try: + self.handle_one_request() + except socket.error as e: + if e[0] != errno.CONNRESET: + raise + + def handle_one_request(self): + self.rfile._sock.setblocking(0) + self.rfile._sock.settimeout(0.2) + SimpleHTTPServer.SimpleHTTPRequestHandler.handle_one_request(self) + + def do_GET(self): + resp = self.server.resp + + self.wfile.write(resp) + self.wfile.flush() + if self.server.unlim: + self.connection.close() + diff --git a/tempesta_fw/t/functional/tests/helpers/cli.py b/tempesta_fw/t/functional/tests/helpers/cli.py new file mode 100644 index 000000000..84d78a4ae --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/cli.py @@ -0,0 +1,22 @@ +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2014-2016 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +# HTTP client emulator. + +# These tests are built around a network of three participants: client, server +# and a running Tempesta FW instance. This module is responsible for the +# client part. +# It allows to connect to the Tempesta and send some data to it in +# various ways. + +from socket import * +from contextlib import contextmanager + +@contextmanager +def connect_to_tfw(port=8081, timeout_sec=5): + socket = create_connection(('127.0.0.1', port), timeout_sec) + yield socket + """It is responsibility of a user of the function to close the socket + """ + after using it. diff --git a/tempesta_fw/t/functional/tests/helpers/conf.py b/tempesta_fw/t/functional/tests/helpers/conf.py new file mode 100644 index 000000000..2df935e66 --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/conf.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies Inc.' +__license__ = 'GPL2' + +import os +from os.path import dirname, realpath +import fileinput +import apache +import subprocess + +be_port = 0 +tempesta_root = "" + +def set_msg_cost(): + p = subprocess.Popen(["sysctl", "-w", "net.core.message_cost=0"], stdout=subprocess.PIPE) + +def set_root(_root): + global tempesta_root + tempesta_root = _root + +def get_root(): + global tempesta_root + if tempesta_root == "": + dir = dirname(realpath(__file__)) + dir = dirname(dir) + dir = dirname(dir) + dir = dirname(dir) + dir = dirname(dir) + root = dirname(dir) + tempesta_root = root + return tempesta_root + +def set_beport(_port): + global be_port + be_port = _port + +def get_beport(): + global be_port + if be_port == 0: + be_port = 8080 + return be_port + +class Config: + name = '' + tmpname = get_root() + '/etc/temp.conf' + + def __init__(self, name, new=True): + self.name = name + if new == False: + open(self.name, "a+") + else: + open(self.name, "w") + return + + def add_option(self, option, value): + with open(self.name, "a+") as conf: + conf.write(option + ' ' + value + ';\n') + + def add_section(self, section): + with open(self.name, "a+") as conf: + conf.write(section + '{\n') + def add_string(self, str): + with open(self.name, "a+") as conf: + conf.write('\n' + str) + + def add_end_of_section(self): + with open(self.name, "a+") as conf: + conf.write('}') + + def del_option(self, option): + temp = open(self.tmpname, 'w') + for line in fileinput.input(self.name): + if not option in line: + temp.write(line) + + os.rename(self.tmpname, self.name) + +class TFWConfig(Config): + def __init__(self): + Config.__init__(self,get_root() + '/etc/tempesta_fw.conf', + new=True) + +class ApacheConfig(Config): + curr_port = 8088 + def __init__(self): + if apache.get_dist() == 'debian': + Config.__init__(self, '/etc/apache2/apache2.conf', + new=False) + else: + Config.__init__(self, '/etc/httpd/conf/httpd.conf', + new=False) + + def add_vhost(self, name): + self.curr_port += 1 + if apache.get_dist() == 'debian': + hconf = Config('/etc/apache2/sites-available/' + name + + '.conf',new=True) + hconf.add_string("") + hconf.add_string("\tDocumentRoot /var/www/sched/html") + hconf.add_string("\tHeader set Vhost: \"" + name + "\"") + hconf.add_string("") + ports = Config('/etc/apache2/ports.conf', new=False) + ports.del_option(str(self.curr_port)) + ports.add_string('\tListen ' + str(self.curr_port)) + apache.link_vhost(name + '.conf') + else: + self.add_string("") + self.add_string("\tDocumentRoot /var/www/sched/html") + self.add_string("\tHeader set Vhost: \"" + name + "\"") + self.add_string("") + self.add_string('Listen ' + str(self.curr_port)) + + return self.curr_port + + diff --git a/tempesta_fw/t/functional/tests/helpers/teardown.py b/tempesta_fw/t/functional/tests/helpers/teardown.py new file mode 100644 index 000000000..45bb141b3 --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/teardown.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +# Functions after executing a test. + +hooks = [] + +def register(func): + global hooks + hooks.append(func) + +def run(): + global hooks + for func in hooks: + if hasattr(func, '__call__'): + func() + +def clean(): + global hooks + hooks = [] diff --git a/tempesta_fw/t/functional/tests/helpers/tfw.py b/tempesta_fw/t/functional/tests/helpers/tfw.py new file mode 100644 index 000000000..251cee603 --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/tfw.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +""" +Helpers for interacting with Tempesta FW (start/stop, configure, etc). +""" + +import os +import subprocess +import sys +import conf + +__author__ = 'NatSys Lab' +__copyright__ = 'Copyright (C) 22014 NatSys Lab. (info@natsys-lab.com).\ +2015-2016 Tempesta Technologies Inc.' +__license__ = 'GPL2' + +_tempesta_dir = conf.get_root() +def start(): + _sh(_tempesta_dir + '/scripts/tempesta.sh --start') + +def start_with_frang(): + _sh(_tempesta_dir + '/scripts/tempesta.sh -f --start') + +def stop(): + _sh("./scripts/tempesta.sh --stop") + +def _sh(command): + return subprocess.check_output(command, shell=True, cwd=_tempesta_dir) + +def _is_started(): + return (0 == subprocess.call("lsmod | grep -q tempesta", shell=True)) + +def _stop_if_started(): + if (_is_started()): + stop() +# Ensure we start and stop in a pristine environment. + assert (not _is_started()) + +def start_bomber(): + _sh("./scripts/tfw_bomber.sh --start") + +def stop_bomber(): + _sh("./scripts/tfw_bomber.sh --stop") +# Sometines we need after a test to remove information about blocked +# ips. +def del_db(): + _sh("rm -rf /opt/tempesta/db") diff --git a/tempesta_fw/t/functional/tests/helpers/tfwparser.py b/tempesta_fw/t/functional/tests/helpers/tfwparser.py new file mode 100644 index 000000000..fdfd8ac02 --- /dev/null +++ b/tempesta_fw/t/functional/tests/helpers/tfwparser.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +__author__ = 'Tempesta Technologies Inc.' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies Inc. (info@natsys-lab.com).' +__license__ = 'GPL2' + +from HTMLParser import HTMLParser +import re +import hashlib +import subprocess +import shlex + +class TFWParser(HTMLParser): + CRLFCRLF = "\r\n\r\n" + status = "" + def get_body_hash(self, data): + body = self.get_body(data) + hasher = hashlib.md5() + hasher.update(body) + hres = hasher.hexdigest() + return hres + + def get_body(self, data): + bstart = data.find(self.CRLFCRLF) + bstart += len(self.CRLFCRLF) + bstop = data.find(self.CRLFCRLF, bstart) + if bstop == -1: + bstop = len(data) + bres = data[bstart:bstop] + return bres + + def set_status(self, data): + status = re.findall("\d\d\d", data) + if len(status) > 0: + self.status = status[0] + + def get_status(self, data): + self.set_status(data) + return int(self.status) + + def check_cache_stat(self): + from_cache = 0 + p1 = subprocess.Popen(shlex.split("cat /proc/tempesta/perfstat"), + stdout=subprocess.PIPE) + p2 = subprocess.Popen(shlex.split("grep cache"), + stdin=p1.stdout, + stdout=subprocess.PIPE) + out = p2.stdout.read() + re_stat = re.findall("\d", out) + from_cache = int(re_stat[0]) + + return from_cache + + def check_log(self, substr): + p1 = subprocess.Popen(shlex.split("dmesg"), + stdout=subprocess.PIPE) + p2 = subprocess.Popen(shlex.split("grep \'" + substr + '\''), + stdin=p1.stdout, + stdout=subprocess.PIPE) + out = p2.stdout.read() + if out.find(substr) > 0: + return True diff --git a/tempesta_fw/t/functional/tests/test_cache.py b/tempesta_fw/t/functional/tests/test_cache.py new file mode 100644 index 000000000..75cbb85c5 --- /dev/null +++ b/tempesta_fw/t/functional/tests/test_cache.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +# #490 Test of restoring of duplicate headers from cache. +# It tests fix of the #482 issue. +# The test adds two duplicate headers in the Apache config and sends +# two requests then checks responses and the cache perf stat just for +# existence a cache event. + +__author__ = 'Tempesta Technologies' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies. (info@natsys-lab.com).' +__license__ = 'GPL2' +import sys +import time +import types + +import conf +import teardown +import tfw +import tfwparser +from socket import * + +class Test: + def __init__(self): + self.res = False + self.vs_get = b"GET / HTTP/1.1\r\n" + self.vs_get += b"Host: localhost\r\nConnection: Keep-alive\r\n" + self.vs_get += "Cookie: session=42\r\n\r\n" + """A problem was with duplicate headers in a response. + So we add two geaders + """ + self.apache_cfg = conf.ApacheConfig() + self.apache_cfg.add_string('Header add Set-Cookie: \"s=42\"') + self.apache_cfg.add_string('Header add Set-Cookie: \"s=42\"') + self.cfg = conf.TFWConfig() + self.cfg.add_option('cache', '2') + self.cfg.add_option('cache_fulfill', '* *') + self.cfg.add_option('listen', '8081') + self.cfg.add_option('server', '127.0.0.1:80') + + + def get_name(self): + return 'Test cache' + + def run(self): + func = tfw._stop_if_started() + teardown.register(func) + body_md5 = "" + status = 0 + self.res = True + parser = tfwparser.TFWParser() + tfw.start() + print('tfw started') + + for x in range(0, 2): + s = socket(AF_INET, SOCK_STREAM) + s.connect(('127.0.0.1', 8081)) + s.sendall(self.vs_get) + data = s.recv(1024) + if x == 0: + body_md5 = parser.get_body_hash(data) + else: + if parser.get_body_hash(data) != body_md5: + print("bodies not match") + self.res = False + status = parser.get_status(data) + if status != 200: + print("status:{}".format(status)) + self.res = False + + s.close() + + stat = parser.check_cache_stat() + if stat == 0: + self.res = False + print("perf stat:from cache:{}".format(stat)) + time.sleep(5) + tfw.stop() + self.apache_cfg.del_option('Header') + print("Res:{}".format(self.res)) diff --git a/tempesta_fw/t/functional/tests/test_frang.py b/tempesta_fw/t/functional/tests/test_frang.py new file mode 100644 index 000000000..4797f6402 --- /dev/null +++ b/tempesta_fw/t/functional/tests/test_frang.py @@ -0,0 +1,644 @@ +#!/usr/bin/env python + +# #375 Test of the Frang classifier. + +# The test consists of a number of tests for different limits of the Frang. +# Each test sets a frang limit to the Tempesta configuration, starts +# the Tempesta with the Frang, then tries a set of requests or a request which +# exceeds +# the limit and detect blocking of the requests or closing of a client +# connection. +# The tests catch an exceptions, but not handle them to enable to run other +# tests. +__author__ = 'Tempesta Technologies' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies. (info@natsys-lab.com).' +__license__ = 'GPL2' +import sys +from os.path import dirname, realpath, sep +import time +import types + +import conf +import tfw +from socket import * +import select +import struct +import socket +import tfwparser +global tcount + +tcount = 0 + +class Test: + + def __init__(self): + self.vs_get = b"GET / HTTP/1.1\r\nhost: loc\r\n\r\n" + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.cfg = conf.TFWConfig() + self.parser = tfwparser.TFWParser() + self.cfg.add_option('cache', '0') + self.cfg.add_option('listen', '8081') + self.cfg.add_option('server', '127.0.0.1:80') + + + def uri_len(self): + """ + The function checks the uri lengh frang limit, so we send + a request that + contains the uri with length greater than limit was set. + If frang blocks + request, the Tempesta returns no data and the Frang writes a warning + to the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_uri_len', '3') + self.cfg.add_end_of_section() + self.vs_get = b"GET /index.html HTTP/1.1\r\nhost: loc\r\n\r\n" + tfw.start_with_frang() + self.s.connect(("127.0.0.1",8081)) + try: + self.s.send(self.vs_get) + data = self.s.recv(1024) + except OSError as e: + pass + if self.parser.check_log("URI length exceeded"): + self.res = True + tcount += 1 + print("uri len:res:{}".format(self.res)) + time.sleep(5) + tfw.stop() + + + def request_rate(self): + """ + The function checks requests per second frang limit. We send a number + of requests from a connection. + After a moment when amount of requests became greater than set limit, + the frang starts bloocking new requests + and client connection is closed,the Frang writes a warning to the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('request_rate', '2') + + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.s.connect(("127.0.0.1",8081)) + + self.vs_get = b"GET / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Connection: Keep-Alive\r\n\r\n" + try: + for x in range(0, 5): + self.s.sendall(self.vs_get) + data = self.s.recv(1024) + + except socket.error as e: + pass + if self.parser.check_log("request rate exceeded"): + self.res = True + tcount += 1 + self.s.close() + print("request rate:res:{}".format(self.res)) + time.sleep(5) + tfw.stop() + + + def request_burst(self): + """ + The function checks the number of requests per fraction of a second + frang limit. + We send a number of requests. When amoun of requests became + greater them + limit that was set, the frang blocks new requests and + the Tempesta closes a client connection. The Frang writes a warning in + the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('request_burst', '1') + + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.s.connect(("127.0.0.1",8081)) + + self.vs_get = b"GET / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Connection: Keep-Alive\r\n\r\n" + try: + for x in range(0, 15): + self.s.sendall(self.vs_get) + data = self.s.recv(1024) + + except socket.error as e: + pass + if self.parser.check_log("requests burst exceeded"): + self.res = True + tcount += 1 + self.s.close() + print("requests burst:res:{}".format(self.res)) + time.sleep(5) + tfw.stop() + + + + def conn_max(self): + """ + The function checks the concurrent connections per client frang limit. + We connecting to the Tempesta from one address, but from different + ports. + After a momet when amount of oconnections becames greater than limit, + the Tempesta seases to accept new connection and the Frang writes a warning to a log. + """ + global tcount + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('ip_block', 'off') + self.cfg.add_option('concurrent_connections', '5') + self.cfg.add_end_of_section() + self.cfg.add_option('keepalive_timeout', '3') + tfw.start_with_frang() + self.vs_get = b"GET / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Connection: Keep-Alive\r\n\r\n" + + try: + socks = [] + for x in range(0,7): + s = socket.socket(AF_INET, SOCK_STREAM) + s.settimeout(2) + s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) + s.connect(("127.0.0.1", 8081)) + s.send(self.vs_get) + data = s.recv(1024) + socks.append(s) + except socket.error as e: + pass + for s in socks: + s.shutdown(SHUT_RDWR) + s.close() + del s + + del socks + if self.parser.check_log("connections max num. exceeded"): + self.res = True + tcount += 1 + print("conn max:res:{}".format(self.res)) + time.sleep(20) + tfw.stop() + + + def conn_rate(self): + """ + The function checks the connections from a client per second + frang limit.We connecting to the Tempesta from different ports + and one address. + When number of connections exceeds limit the frang will block + new connections and write a warning to the log. + """ + global tcount + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('ip_block', 'off') + self.cfg.add_option('connection_rate', '5') + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.vs_get = b"GET / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Connection: Keep-Alive\r\n\r\n" + + socks = [] + try: + for x in range(0,7): + s = socket.socket(AF_INET, SOCK_STREAM) + s.settimeout(2) + s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) + s.connect(("127.0.0.1", 8081)) + socks.append(s) + + except socket.error as e: + pass + if self.parser.check_log("new connections rate exceeded"): + self.res = True + tcount += 1 + + print("conn rate:res:{}".format(self.res)) + for s in socks: + s.shutdown(SHUT_RDWR) + s.close() + del s + del socks + time.sleep(20) + tfw.stop() + + + def ct_required(self): + """ + The function checks the presence of the "Content-Type" header frang + limit. + We set the ct_required frang limit and send a request without the header + "Content-Type" and expect the frang blocks request, writes a warning + to the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_ct_required', 'true') + self.cfg.add_end_of_section() + self.vs_get = b"POST / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Connection: Keep-alive\r\n\r\n" + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(self.vs_get) + data = self.s.recv(1024) + if self.parser.check_log("Content-Type header field"): + self.res = True + tcount += 1 + + print("ct required:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def conn_burst(self): + """ + The function checks the connections per a fraction of a second + frang limit. + We make a number of connections from different ports of one address. + When amount of connections reaches the limit, the Frang writes a warning + at the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('ip_block', 'off') + self.cfg.add_option('connection_burst', '1') + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.vs_get = b"GET / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Connection: Keep-Alive\r\n\r\n" + + try: + socks = [] + conncount = 0 + for x in range(0,7): + s = socket.socket(AF_INET, SOCK_STREAM) + s.settimeout(2) + s.connect(("127.0.0.1", 8081)) + socks.append(s) + + except socket.error as e: + pass + if self.parser.check_log("new connections burst exceeded"): + self.res = True + tcount += 1 + for s in socks: + s.close() + del s + print("conn burst:res:{}".format(self.res)) + + time.sleep(5) + tfw.stop() + + + def host_required(self): + """ +# The function checks the presence of the "Host" header frang limit. +# We set the limit and send a requests without "Host" header. +# So the frang block +# the request and the Frang writes a warning at the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_host_required', 'true') + self.cfg.add_end_of_section() + self.vs_get = b"POST / HTTP/1.1\r\n" + self.vs_get += b"Connection: Keep-alive\r\n\r\n" + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(self.vs_get) + data = self.s.recv(1024) + if self.parser.check_log("Host header field"): + self.res = True + tcount += 1 + + print("host required:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def body_len(self): + """ + The function checks the length of request`s body frang limit. + We set the limit and send a request with a body greater than the limit. + So the frang blocks the request and writes a warning at the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_ct_vals', '[\"text/html\"]') + self.cfg.add_option('http_body_len', '10') + self.cfg.add_end_of_section() + self.vs_get = b"POST /a.html HTTP/1.1\r\nHost: loc\r\n" + self.vs_get += b"Content-Type: text/html\r\n" + self.vs_get += b"Content-Length: 20\r\n\r\n" + self.vs_get += b"content\r\n\r\n" + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(self.vs_get) + data = self.s.recv(1024) + if self.parser.check_log("body length exceeded"): + self.res = True + tcount += 1 + + print("body len:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def field_len(self): + """ + The function checks the header field length frang limit. We set + the limit and send a request with a header greater than limit. + The frang blocks + the request and the Frang writes a warning at the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_field_len', '10') + self.cfg.add_end_of_section() + self.vs_get = b"POST /a.html HTTP/1.1\r\nHost: loc\r\n" + self.vs_get += b"Content-Type: application/xml\r\n" + self.vs_get += b"Content-Length: 20\r\n\r\n" + self.vs_get += b"content\r\n\r\n" + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(self.vs_get) + data = self.s.recv(1024) + if self.parser.check_log("field length exceeded"): + self.res = True + tcount += 1 + + print("field len:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def ct_vals(self): + """ + The function checks the permitted values of the "Content-Type" + header frang + limit. + We add a set of the "Content-Type" permitted values to the frang + section. + And then send a request with a "Content-Type" header value witch is not + present in the added set. The frang blocks the request and the Frang + writes a warning at the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_ct_vals', '[\"text/html\"]') + self.cfg.add_end_of_section() + self.vs_get = b"POST / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Content-type: application/xml\r\n\r\n" + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(self.vs_get) + data = self.s.recv(1024) + if self.parser.check_log("restricted Content-Type"): + self.res = True + tcount += 1 + + print("ct vals:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def req_method(self): + """ + The function checks the permitted request methods frang limit. + We add a set of request method to the frang limits and send + a request with arequest method + witch is not present in the set. So the frang have to block the request + and write a warning to the log. + """ + global tcount + + self.res = False + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_methods', 'get') + self.cfg.add_end_of_section() + self.vs_get = b"POST / HTTP/1.1\r\nhost: loc\r\n" + self.vs_get += b"Content-type: application/xml\r\n\r\n" + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(self.vs_get) + data = self.s.recv(1024) + if self.parser.check_log("restricted HTTP method"): + self.res = True + tcount += 1 + + print("request method:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def header_chunks(self): + """ + The function checks the amount of a header chunks frang limit. We set + the limit and divide request`s headers into parts and send parts + separate. + So the frang blocks the request and writes a warning at the log. + """ + global tcount + part1 = b'GET / HTTP/1.1\r\n' + part2 = b'host: loc\r\n' + part3 = b'Connection: close\r\n\r\n' + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_header_chunk_cnt', '1') + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.setsockopt(socket.IPPROTO_TCP, TCP_NODELAY, 1) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(part1) + self.s.send(part2) + self.s.send(part3) + data = self.s.recv(1024) + if self.parser.check_log("header chunk count exceeded"): + self.res = True + tcount += 1 + self.s.close() + time.sleep(5) + tfw.stop() + print("header chunks:res:{}".format(self.res)) + + + + def header_timeout(self): + """ + The function checks the timeout between a request header chunks + frang limit. + We set the limit, then send a request header divided into parts with a pause + between the parts. The frang blocks the request and + writes a warning at the log. + """ + global tcount + part1 = b'GET / HTTP/1.1\r\n' + part2 = b'host: loc\r\n\r\n' + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('client_header_timeout', '1') + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.setsockopt(socket.IPPROTO_TCP, TCP_NODELAY, 1) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(part1) + time.sleep(1) + self.s.send(part2) + data = self.s.recv(1024) + if len(data) == 0: + tcount += 1 + self.res = True + + print( "header timeout:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def body_timeout(self): + """ + The function checks the timeout between body chunks frang limit. + We set the limit and divide our request into parts that a body is + divided. + Then send the parts with a pause. The frang has to bock the request + and writes a warning at the log. + """ + global tcount + self.res = False + part1 = b"POST /a.html HTTP/1.1\r\nHost: loc\r\n" + part1 += b"Content-Type: text/html\r\nContent-Length: 20" + part1 += b"\r\n\r\ncontent" + part2 = b"\r\n\r\n" + + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_ct_vals', '[\"text/html\"]') + self.cfg.add_option('client_body_timeout', '1') + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.setsockopt(socket.IPPROTO_TCP, TCP_NODELAY, 1) + self.s.connect(('127.0.0.1', 8081)) + self.s.send(part1) + time.sleep(2) + self.s.send(part2) + data = self.s.recv(1024) + if self.parser.check_log("body timeout exceed"): + tcount += 1 + self.res = True + + print("body timeout:res:{}".format(self.res)) + self.s.close() + time.sleep(5) + tfw.stop() + + + def body_chunks(self): + """ + The function checks the amount of a body chunks frang limit. We set + the limit. + Then divide our request into parts that the body of the request was + divided + too And send the parts with a pause. The frang has to block the request. + and to write a warnining at the log. + """ + global tcount + self.res = False + part1 = b"POST /a.html HTTP/1.1\r\nHost: loc\r\n\ +Content-Type: text/html\r\nContent-Length: 30\r\n\r\ncontent" + part2 = b"" + part3 = b"\r\n\r\n" + + self.__init__() + self.cfg.add_section('frang_limits') + self.cfg.add_option('http_ct_vals', '[\"text/html\"]') + self.cfg.add_option('http_body_chunk_cnt', '1') + self.cfg.add_end_of_section() + tfw.start_with_frang() + self.s = socket.socket(AF_INET, SOCK_STREAM) + self.s.setsockopt(socket.IPPROTO_TCP, TCP_NODELAY, 1) + self.s.connect(('127.0.0.1', 8081)) + try: + self.s.send(part1) + self.s.send(part2) + self.s.send(part3) + data = self.s.recv(1024) + except socket.error as e: + pass + + if self.parser.check_log("body chunk count exceed"): + self.res = True + tcount += 1 + + print("body chunks:res:{}".format(self.res)) + time.sleep(5) + tfw.stop() + + def get_name(self): + return 'test Frang' + + def run(self): + global tcount + conf.set_msg_cost() + tests = [self.request_burst(), self.body_chunks(), + self.header_chunks(), self.body_timeout(), + self.uri_len(), self.field_len(), self.body_len(), + self.host_required(), self.ct_required(), + self.ct_vals(), self.conn_rate(), self.req_method(), + self.request_rate(), self.conn_max(), self.conn_rate()] + for f in tests: + if hasattr(f, '__call__'): + f() + + print("tests:{}/15".format(tcount)) + diff --git a/tempesta_fw/t/functional/tests/test_parser.py b/tempesta_fw/t/functional/tests/test_parser.py new file mode 100644 index 000000000..445fa4a53 --- /dev/null +++ b/tempesta_fw/t/functional/tests/test_parser.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python + +# #629 A set of tests for responses without the "Content-Length:" header. +# In this set we using be.py as backend server to full control of the responses.# We set a response for the backend, send a request to the Tempesta, +# get a ressponse from the Tempesta and check it. +# For a test with cache we also check the cache perf stat. + +__author__ = 'Tempesta Technologies' +__copyright__ = 'Copyright (C) 2016 Tempesta Technologies. (info@natsys-lab.com).' +__license__ = 'GPL2' + +import sys + +import socket +import tfw +import conf +import be +import tfwparser +import datetime +import os +import requests + +class Test: + def __init__(self): + self.res = "" + self.cfg = conf.TFWConfig() + self.cfg.add_option('cache', '0') + self.cfg.add_option('listen', '8081') + self.cfg.add_option('server', '127.0.0.1:' + + str(conf.get_beport())) + + def run_with_cache(self): + """Send request twice and check responses. + """ + self.res = "cache - two queries and compare bodies:\n" + self.cfg.del_option('cache') + self.cfg.add_option('cache', '1') + self.cfg.add_option('cache_fulfill', '* *') + + self.run_test(2) + parser = tfwparser.TFWParser() + + + + def run_no_length_no_body(self): + """A response without the Content-Length header and without + + a body. + It checks state of the #629 issue. + """ + self.res = "no Content-Length header and no body:\n" + + + self.run_test(1) + + def run_no_length_body(self): + """A test without the Content-Length header, but with a body in + a response. + """ + + self.res = "no Content-Length header, but is body:\n" + + self.run_test(1) + def run_test(self, num): + + vs_get = b"GET / HTTP/1.1\nHost: loc\n" +\ + b"Connection: Keep-Alive\r\n\r\n" + parser = tfwparser.TFWParser() + body_hash = parser.get_body_hash(self.resp) + + tfw.start() + i = 0 + while i < num: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + data = "" + s.connect(("127.0.0.1",8081)) + s.settimeout(1) + try: + s.sendall(vs_get) + data = s.recv(1024) + except socket.error as e: + self.res = "exception:{}".format(e) + print("exception:{}".format(e)) + + s.close() + i += 1 + if len(data) > 0: + parser = tfwparser.TFWParser() + if len(parser.get_body(data)) == 0: + self.res += 'no body\n' + else: + b_hash = parser.get_body_hash(data) + if b_hash != body_hash: + self.res += "bodies " + self.res += "not equals\n" + else: + self.res += "bodies equals\n" + status = parser.get_status(data) + self.res += "status:{}\n".format(status) + if num > 1: + cache_stat = parser.check_cache_stat() + self.res +="perf stat: from cache=" + self.res += str(cache_stat) + '\n' + tfw.stop() + + print(self.res) + + def run(self): + self.resp = b'HTTP/1.1' + b' 200 - OK\n' + date = datetime.datetime.utcnow().strftime("%a, %d %b %Y" +\ + " %H:%M:%S GMT") + self.resp += b"Date: " + date + b"\r\n" + self.resp += b"Server: be python\r\n" + self.resp += b'\r\ncontent\r\n\r\n' + be_pid = be.start(True, self.resp) + self.run_no_length_body() + self.run_no_length_no_body() + self.run_with_cache() + be.stop(be_pid) + + def get_name(self): + return 'test_unlimited' +