feat(logs): add logs dashboard

Ion.egg-info/SOURCES.txt

@@ -569,6 +569,8 @@ intranet/apps/logs/__init__.py
 intranet/apps/logs/admin.py
 intranet/apps/logs/forms.py
 intranet/apps/logs/models.py
+intranet/apps/logs/urls.py
+intranet/apps/logs/views.py
 intranet/apps/logs/migrations/0001_initial.py
 intranet/apps/logs/migrations/0002_auto_20230613_1759.py
 intranet/apps/logs/migrations/0003_request_request.py
@@ -906,6 +908,7 @@ intranet/static/css/groups.scss
 intranet/static/css/hoco_ribbon.scss
 intranet/static/css/hoco_scores.scss
 intranet/static/css/login.scss
+intranet/static/css/logs.scss
 intranet/static/css/lostfound.scss
 intranet/static/css/mobile.scss
 intranet/static/css/oauth.scss
@@ -1126,6 +1129,7 @@ intranet/static/js/files.js
 intranet/static/js/hoco_ribbon.js
 intranet/static/js/hoco_scores.js
 intranet/static/js/login.js
+intranet/static/js/logs.js
 intranet/static/js/polls-vote.js
 intranet/static/js/polls.js
 intranet/static/js/responsive.core.js
@@ -3445,6 +3449,10 @@ intranet/templates/itemreg/item_view.html
 intranet/templates/itemreg/register_delete.html
 intranet/templates/itemreg/register_form.html
 intranet/templates/itemreg/search.html
+intranet/templates/logs/home.html
+intranet/templates/logs/pagination.html
+intranet/templates/logs/query.html
+intranet/templates/logs/request.html
 intranet/templates/logs/admin/flag_request.html
 intranet/templates/lostfound/founditem.html
 intranet/templates/lostfound/founditem_delete.html

docs/sourcedoc/intranet.apps.logs.rst

@@ -28,6 +28,22 @@ intranet.apps.logs.models module
    :undoc-members:
    :show-inheritance:
 
+intranet.apps.logs.urls module
+------------------------------
+
+.. automodule:: intranet.apps.logs.urls
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+intranet.apps.logs.views module
+-------------------------------
+
+.. automodule:: intranet.apps.logs.views
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

intranet/apps/logs/admin.py

@@ -154,7 +154,7 @@ def truncated_path(self):
     actions = ["flag_requests"]
 
     def request_json(self, obj):
-        return json.dumps(json.loads(obj.request), indent=4, sort_keys=True)
+        return json.dumps(json.loads(obj.request), indent=4, sort_keys=True).replace('\\"', "'")
 
     @admin.action(description="Flag selected requests for review")
     def flag_requests(self, request, queryset):

intranet/apps/logs/models.py

@@ -1,3 +1,5 @@
+import json
+
 from django.conf import settings
 from django.db import models
 
@@ -20,12 +22,18 @@ class Request(models.Model):
 
     @property
     def username(self):
-        if self.user:
-            return self.user.username
-        return "unknown_user"
+        return self.user.username if self.user else "anonymous"
+
+    @property
+    def request_json(self):
+        return json.dumps(json.loads(self.request), indent=4, sort_keys=True).replace('\\"', "'")
+
+    @property
+    def request_json_obj(self):
+        return json.loads(self.request)
 
     def __str__(self):
-        return f"""{self.ip} - {self.user} - [{self.timestamp}] "{self.path}" "{self.user_agent}" """
+        return f'{self.timestamp.astimezone(settings.PYTZ_TIME_ZONE).strftime("%b %d %Y %H:%M:%S")} - {self.username} - {self.ip} - {self.method} "{self.path}"'  # noqa: E501 pylint: disable=line-too-long
 
     class Meta:
         ordering = ["-timestamp"]

intranet/apps/logs/urls.py

@@ -0,0 +1,8 @@
+from django.urls import re_path
+
+from . import views
+
+urlpatterns = [
+    re_path(r"^$", views.logs_view, name="logs"),
+    re_path(r"^/request/(?P<request_id>\d+)$", views.request_view, name="request"),
+]

intranet/apps/logs/views.py

@@ -0,0 +1,204 @@
+import datetime
+import ipaddress
+
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.http import Http404
+from django.shortcuts import get_object_or_404, render
+from django.utils.timezone import make_aware
+
+from ..auth.decorators import reauthentication_required
+from ..users.models import User
+from .models import Request
+
+DISPLAY_NUM = 500
+PAGE_RANGE = 5
+
+HTTP_METHODS = [
+    "GET",
+    "POST",
+    "PUT",
+    "DELETE",
+    "PATCH",
+    "HEAD",
+    "OPTIONS",
+    "CONNECT",
+    "TRACE",
+]
+
+TEXT_SEARCH_TYPES = [
+    ("equals", "Equals"),
+    ("contains", "Contains"),
+    ("starts", "Starts with"),
+    ("ends", "Ends with"),
+]
+
+
+def logs_context(queryset, request):
+    total = queryset.count()
+    last = total // DISPLAY_NUM + 1
+
+    try:
+        page = int(request.GET.get("page", 1))
+        page = min(max(1, page), last)
+    except ValueError:
+        page = 1
+
+    start = (page - 1) * DISPLAY_NUM
+    end = page * DISPLAY_NUM
+    rqs = queryset.prefetch_related("user")[start:end]
+
+    context = {
+        "rqs": rqs,
+        "total": total,
+        "start": start + 1 if total > 0 else 0,
+        "end": min(end, total),
+        "last_page": last,
+        "current_page": page,
+        "show_first_page": page > PAGE_RANGE + 1,
+        "show_first_page_dot": page > PAGE_RANGE + 2,
+        "show_last_page": page < last - PAGE_RANGE,
+        "show_last_page_dot": page < last - PAGE_RANGE - 1,
+    }
+
+    for i in range(1, PAGE_RANGE + 1):
+        context[f"next_page_{i}"] = page + i if page + i <= last else None
+        context[f"prev_page_{i}"] = page - i if page - i >= 1 else None
+
+    return context
+
+
+@login_required
+@reauthentication_required
+def logs_view(request):
+    if not request.user.is_global_admin:
+        raise Http404
+
+    context = {
+        "all_users": User.objects.order_by("username").all(),
+        "all_methods": HTTP_METHODS,
+        "text_search_types": TEXT_SEARCH_TYPES,
+    }
+
+    queries = {}
+
+    if request.GET.get("user", None):
+        users = set(request.GET.getlist("user"))
+        context["selected_users"] = users.copy()
+
+        if "anonymous" in users:
+            users.remove("anonymous")
+            queries["user__isnull"] = True
+
+            if len(users) > 0:
+                messages.warning(request, "Searching only for anonymous users. Ignoring other users.")
+
+        else:
+            users = User.objects.filter(username__in=users)
+
+            if users.count() == 1:
+                queries["user"] = users.first()
+            elif users.count() > 1:
+                queries["user__in"] = users
+
+    if request.GET.get("ip", None):
+        ips = set(request.GET.getlist("ip"))
+        context["selected_ips"] = ips.copy()
+        to_expand = [ip for ip in ips if "/" in ip]
+
+        for ip in to_expand:
+            try:
+                network = ipaddress.ip_network(ip, strict=False)
+                ips.remove(ip)
+
+                if network.num_addresses > 2**16:
+                    messages.error(request, f"Subnet too large: {ip}.")
+                else:
+                    ips |= set(str(ip) for ip in network.hosts())
+
+            except ValueError:
+                messages.error(request, f"Invalid IP network: {ip}")
+
+        if len(ips) == 1:
+            queries["ip"] = ips.pop()
+        else:
+            queries["ip__in"] = ips
+
+    if request.GET.get("method", None):
+        selected_methods = set(request.GET.getlist("method"))
+        context["selected_methods"] = selected_methods.copy()
+        if len(selected_methods) == 1:
+            queries["method"] = selected_methods.pop()
+        else:
+            queries["method__in"] = selected_methods
+
+    if request.GET.get("from", None):
+        from_time = request.GET["from"]
+        context["selected_from"] = from_time
+        try:
+            from_time = datetime.datetime.strptime(from_time, "%Y-%m-%d %H:%M:%S")
+            from_time = make_aware(from_time)
+            queries["timestamp__gte"] = from_time
+        except ValueError:
+            messages.error(request, "Invalid from time.")
+
+    if request.GET.get("to", None):
+        to_time = request.GET["to"]
+        context["selected_to"] = to_time
+        try:
+            to_time = datetime.datetime.strptime(to_time, "%Y-%m-%d %H:%M:%S")
+            to_time = make_aware(to_time)
+            queries["timestamp__lte"] = to_time
+        except ValueError:
+            messages.error(request, "Invalid to time.")
+        context["selected_to"] = request.GET["to"]
+
+    if request.GET.get("path-type", None):
+        path_type = request.GET["path-type"]
+        context["selected_path_type"] = path_type
+
+        if request.GET.get("path", None):
+            path = request.GET["path"]
+            context["selected_path"] = path
+
+            if path_type == "contains":
+                queries["path__contains"] = path
+            elif path_type == "starts":
+                queries["path__startswith"] = path
+            elif path_type == "ends":
+                queries["path__endswith"] = path
+            else:
+                queries["path"] = path
+
+    if request.GET.get("user-agent-type", None):
+        user_agent_type = request.GET["user-agent-type"]
+        context["selected_user_agent_type"] = user_agent_type
+
+        if request.GET.get("user-agent", None):
+            user_agent = request.GET["user-agent"]
+            context["selected_user_agent"] = user_agent
+
+            if user_agent_type == "contains":
+                queries["user_agent__contains"] = user_agent
+            elif user_agent_type == "starts":
+                queries["user_agent__startswith"] = user_agent
+            elif user_agent_type == "ends":
+                queries["user_agent__endswith"] = user_agent
+            else:
+                queries["user_agent"] = user_agent
+
+    queryset = Request.objects.filter(**queries)
+    context |= logs_context(queryset, request)
+
+    return render(request, "logs/home.html", context)
+
+
+@login_required
+@reauthentication_required
+def request_view(request, request_id):
+    if not request.user.is_global_admin:
+        raise Http404
+
+    rq = get_object_or_404(Request, id=request_id)
+
+    return render(request, "logs/request.html", {"rq": rq})

intranet/middleware/access_log.py

@@ -70,13 +70,13 @@ def __call__(self, request):
             # Redact passwords
             r_request = json.loads(r.request)
             if "password" in r_request["POST"]:
-                r_request["POST"]["password"] = "********"
+                r_request["POST"]["password"] = "***REDACTED***"
                 r.request = json.dumps(r_request)
                 r.save()
             if "password" in r_request["body"]:
                 idx = r_request["body"].index("password=")
                 length = r_request["body"][idx:].index("&") if "&" in r_request["body"][idx:] else len(r_request["body"][idx:])
-                r_request["body"] = r_request["body"][:idx] + "password=********" + r_request["body"][idx + length:]
+                r_request["body"] = r_request["body"][:idx] + "password=***REDACTED***" + r_request["body"][idx + length:]
                 r.request = json.dumps(r_request)
                 r.save()
 

intranet/settings/__init__.py

@@ -6,6 +6,7 @@
 from typing import Any, Dict, List, Tuple  # noqa
 
 import celery.schedules
+import pytz
 import sentry_sdk
 from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
@@ -241,6 +242,7 @@
 # although not all choices may be available on all operating systems.
 # In a Windows environment this must be set to your system time zone.
 TIME_ZONE = "America/New_York"
+PYTZ_TIME_ZONE = pytz.timezone(TIME_ZONE)
 
 # Language code for this installation. All choices can be found here:
 # http://www.i18nguy.com/unicode/language-identifiers.html
@@ -358,6 +360,7 @@
     "signage.page",
     "courses",
     "sessionmgmt",
+    "logs",
     "dark/base",
     "dark/login",
     "dark/schedule",
@@ -930,7 +933,7 @@ def get_log(name):  # pylint: disable=redefined-outer-name; 'name' is used as th
 # The Referrer-policy header
 REFERRER_POLICY = "strict-origin-when-cross-origin"
 
-REAUTHENTICATION_EXPIRE_TIMEOUT = 2 * 60 * 60  # seconds
+REAUTHENTICATION_EXPIRE_TIMEOUT = 15 * 60  # seconds
 
 EIGHTH_COORDINATOR_NAME = "Laura Slonina"