Merge: + QueryLog: server-side paging and filtering

Close #776

* commit 'f04c028e380182558d772ac6d65efe4f7bdb48d8':
  + client: handle filter change
  + client: server side pagination for the query logs
  * querylog: POST /control/querylog
  * openapi

AGHTechDoc.md

@@ -43,6 +43,7 @@ Contents:
 	* API: Set statistics parameters
 	* API: Get statistics parameters
 * Query logs
+	* API: Get query log
 	* API: Set querylog parameters
 	* API: Get querylog parameters
 * Filtering
@@ -1007,6 +1008,92 @@ Response:
 
 ## Query logs
 
+When a new DNS request is received and processed, we store information about this event in "query log".  It is a file on disk in JSON format:
+
+	{
+	"Question":"...","
+	Answer":"...",
+	"Result":{
+		"IsFiltered":true,
+		"Reason":3,
+		"Rule":"...",
+		"FilterID":1
+		},
+	"Time":"...",
+	"Elapsed":12345,
+	"IP":"127.0.0.1"
+	}
+
+
+### Adding new data
+
+First, new data is stored in a memory region.  When this array is filled to a particular amount of entries (e.g. 5000), we flush this data to a file and clear the array.
+
+
+### Getting data
+
+When UI asks for data from query log (see "API: Get query log"), server reads the newest entries from memory array and the file.  The maximum number of items returned per one request is limited by configuration.
+
+
+### Removing old data
+
+We store data for a limited amount of time - the log file is automatically rotated.
+
+
+### API: Get query log
+
+Request:
+
+	POST /control/querylog
+
+	{
+	older_than: "2006-01-02T15:04:05.999999999Z07:00" // must be "" for the first request
+
+	filter:{
+		domain: "..."
+		client: "..."
+		question_type: "A" | "AAAA"
+		response_status: "" | "filtered"
+	}
+	}
+
+If `older_than` value is set, server returns the next chunk of entries that are older than this time stamp.  This setting is used for paging.  UI sets this value to `""` on the first request and gets the latest log entries.  To get the older entries, UI sets this value to the timestamp of the last (the oldest) entry from the previous response from Server.
+
+If "filter" settings are set, server returns only entries that match the specified request.
+
+For `filter.domain` and `filter.client` the server matches substrings by default: `adguard.com` matches `www.adguard.com`.  Strict matching can be enabled by enclosing the value in double quotes: `"adguard.com"` matches `adguard.com` but doesn't match `www.adguard.com`.
+
+Response:
+
+	[
+	{
+		"answer":[
+			{
+			"ttl":10,
+			"type":"AAAA",
+			"value":"::"
+			}
+			...
+		],
+		"client":"127.0.0.1",
+		"elapsedMs":"0.098403",
+		"filterId":1,
+		"question":{
+			"class":"IN",
+			"host":"doubleclick.net",
+			"type":"AAAA"
+		},
+		"reason":"FilteredBlackList",
+		"rule":"||doubleclick.net^",
+		"status":"NOERROR",
+		"time":"2006-01-02T15:04:05.999999999Z07:00"
+	}
+	...
+	]
+
+The most recent entries are at the top of list.
+
+
 ### API: Set querylog parameters
 
 Request:

client/package.json

@@ -17,7 +17,7 @@
     "i18next-browser-languagedetector": "^2.2.3",
     "lodash": "^4.17.15",
     "nanoid": "^1.2.3",
-    "prop-types": "^15.6.1",
+    "prop-types": "^15.7.2",
     "react": "^16.4.0",
     "react-click-outside": "^3.0.1",
     "react-dom": "^16.4.0",
@@ -27,7 +27,7 @@
     "react-redux-loading-bar": "^4.0.7",
     "react-router-dom": "^4.2.2",
     "react-router-hash-link": "^1.2.2",
-    "react-table": "^6.8.6",
+    "react-table": "^6.10.3",
     "react-transition-group": "^2.4.0",
     "redux": "^4.0.0",
     "redux-actions": "^2.4.0",

client/src/__locales/en.json

@@ -163,7 +163,6 @@
     "show_filtered_type": "Show filtered",
     "no_logs_found": "No logs found",
     "refresh_btn": "Refresh",
-    "last_dns_queries": "Last 5000 DNS queries",
     "previous_btn": "Previous",
     "next_btn": "Next",
     "loading_table_status": "Loading...",
@@ -182,6 +181,7 @@
     "query_log_enable": "Enable log",
     "query_log_configuration": "Logs configuration",
     "query_log_disabled": "The query log is disabled and can be configured in the <0>settings</0>",
+    "query_log_strict_search": "Use double quotes for strict search",
     "source_label": "Source",
     "found_in_known_domain_db": "Found in the known domains database.",
     "category_label": "Category",

client/src/actions/queryLogs.js

@@ -4,15 +4,19 @@ import apiClient from '../api/Api';
 import { addErrorToast, addSuccessToast } from './index';
 import { normalizeLogs } from '../helpers/helpers';
 
+export const setLogsPagination = createAction('LOGS_PAGINATION');
+export const setLogsFilter = createAction('LOGS_FILTER');
+
 export const getLogsRequest = createAction('GET_LOGS_REQUEST');
 export const getLogsFailure = createAction('GET_LOGS_FAILURE');
 export const getLogsSuccess = createAction('GET_LOGS_SUCCESS');
 
-export const getLogs = () => async (dispatch) => {
+export const getLogs = config => async (dispatch) => {
     dispatch(getLogsRequest());
     try {
-        const logs = normalizeLogs(await apiClient.getQueryLog());
-        dispatch(getLogsSuccess(logs));
+        const { filter, lastRowTime: older_than } = config;
+        const logs = normalizeLogs(await apiClient.getQueryLog({ filter, older_than }));
+        dispatch(getLogsSuccess({ logs, ...config }));
     } catch (error) {
         dispatch(addErrorToast({ error }));
         dispatch(getLogsFailure(error));

client/src/api/Api.js

@@ -482,14 +482,18 @@ class Api {
     }
 
     // Query log
-    GET_QUERY_LOG = { path: 'querylog', method: 'GET' };
+    GET_QUERY_LOG = { path: 'querylog', method: 'POST' };
     QUERY_LOG_CONFIG = { path: 'querylog_config', method: 'POST' };
     QUERY_LOG_INFO = { path: 'querylog_info', method: 'GET' };
     QUERY_LOG_CLEAR = { path: 'querylog_clear', method: 'POST' };
 
-    getQueryLog() {
+    getQueryLog(data) {
         const { path, method } = this.GET_QUERY_LOG;
-        return this.makeRequest(path, method);
+        const config = {
+            data,
+            headers: { 'Content-Type': 'application/json' },
+        };
+        return this.makeRequest(path, method, config);
     }
 
     getQueryLogInfo() {

client/src/components/Logs/Logs.css

@@ -107,6 +107,11 @@
     border: 1px solid rgba(0, 40, 100, 0.12);
 }
 
+.logs__table .rt-thead.-filters select {
+    background: #fff url("data:image/svg+xml;base64,PHN2ZyB4bWxucz0naHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmcnIHZpZXdCb3g9JzAgMCAxMCA1Jz48cGF0aCBmaWxsPScjOTk5JyBkPSdNMCAwTDEwIDBMNSA1TDAgMCcvPjwvc3ZnPg==") no-repeat right 0.75rem center;
+    background-size: 8px 10px;
+}
+
 .logs__table .rt-thead.-filters input:focus,
 .logs__table .rt-thead.-filters select:focus {
     border-color: #1991eb;
@@ -130,6 +135,21 @@
     overflow: hidden;
 }
 
+.logs__input-wrap {
+    position: relative;
+}
+
+.logs__notice {
+    position: absolute;
+    z-index: 1;
+    top: 8px;
+    right: 10px;
+    margin-top: 3px;
+    font-size: 12px;
+    text-align: left;
+    color: #a5a5a5;
+}
+
 .logs__whois {
     display: inline;
 }