Merge pull request #170 from getappmap/sw/feat/record_requests__version3

feat: Save an AppMap per request (record_requests)

README.md

@@ -32,7 +32,7 @@ Clone the repo to begin development. Note that vendored dependencies are include
 submodules.
 
 ```shell
-% g clone --recurse-submodules https://github.com/applandinc/appmap-python.git
+% git clone --recurse-submodules https://github.com/applandinc/appmap-python.git
 Cloning into 'appmap-python'...
 remote: Enumerating objects: 167, done.
 remote: Counting objects: 100% (167/167), done.

appmap/_implementation/env.py

@@ -76,6 +76,10 @@ def enabled(self, value):
     def display_params(self):
         return self.get("APPMAP_DISPLAY_PARAMS", "true").lower() == "true"
 
+    @property
+    def record_all_requests(self):
+        return self.get("APPMAP_RECORD_REQUESTS", "false").lower() == "true"
+
     def _configure_logging(self):
         log_level = self.get("APPMAP_LOG_LEVEL", "warning").upper()
 

appmap/_implementation/testing_framework.py

@@ -1,14 +1,18 @@
 """Shared infrastructure for testing framework integration."""
 
-import os
 import re
 from contextlib import contextmanager
-from hashlib import sha256
-from tempfile import NamedTemporaryFile
 
 import inflection
 
-from appmap._implementation import configuration, env, generation, recording
+from appmap._implementation import (
+    configuration,
+    env,
+    generation,
+    recording,
+    web_framework,
+)
+from appmap._implementation.env import Env
 from appmap._implementation.utils import fqname
 
 from .metadata import Metadata
@@ -91,36 +95,6 @@ def metadata(self):
         return ret
 
 
-NAME_MAX = 255  # true for most filesystems
-HASH_LEN = 7  # arbitrary, but git proves it's a reasonable value
-APPMAP_SUFFIX = ".appmap.json"
-
-
-def name_hash(namepart):
-    """Returns the hex digits of the sha256 of the os.fsencode()d namepart."""
-    return sha256(os.fsencode(namepart)).hexdigest()
-
-
-def write_appmap(basedir, basename, contents):
-    """Write an appmap file into basedir.
-
-    Adds APPMAP_SUFFIX to basename; shortens the name if necessary.
-    Atomically replaces existing files. Creates the basedir if required.
-    """
-
-    if len(basename) > NAME_MAX - len(APPMAP_SUFFIX):
-        part = NAME_MAX - len(APPMAP_SUFFIX) - 1 - HASH_LEN
-        basename = basename[:part] + "-" + name_hash(basename[part:])[:HASH_LEN]
-    filename = basename + APPMAP_SUFFIX
-
-    if not basedir.exists():
-        basedir.mkdir(parents=True, exist_ok=True)
-
-    with NamedTemporaryFile(mode="w", dir=basedir, delete=False) as tmp:
-        tmp.write(contents)
-    os.replace(tmp.name, basedir / filename)
-
-
 class session:
     def __init__(self, name, recorder_type, version=None):
         self.name = name
@@ -155,7 +129,9 @@ def record(self, klass, method, **kwds):
                 yield metadata
         finally:
             basedir = env.Env.current.output_dir / self.name
-            write_appmap(basedir, item.filename, generation.dump(rec, metadata))
+            web_framework.write_appmap(
+                basedir, item.filename, generation.dump(rec, metadata)
+            )
 
 
 @contextmanager

appmap/_implementation/utils.py

@@ -1,5 +1,6 @@
 import inspect
 import os
+import re
 import shlex
 import subprocess
 import threading
@@ -192,3 +193,10 @@ def _wrap_cls(patch):
         return patch
 
     return _wrap_cls
+
+
+# this is different than appmap-ruby: part of its logic is in write_appmap
+def scenario_filename(name, separator="_"):
+    pattern = r"[^a-z0-9\-_]+"
+    replacement = separator
+    return re.sub(pattern, replacement, name, flags=re.IGNORECASE)

appmap/_implementation/web_framework.py

@@ -1,11 +1,19 @@
 """Common utilities for web framework integration"""
 
+import datetime
+import os
+import os.path
 import re
 import time
+from abc import abstractmethod
+from hashlib import sha256
+from tempfile import NamedTemporaryFile
 
-from appmap._implementation.event import Event, ReturnEvent, describe_value
+from appmap._implementation import generation
+from appmap._implementation.env import Env
+from appmap._implementation.event import Event, ReturnEvent, _EventIds, describe_value
 from appmap._implementation.recording import Recorder
-from appmap._implementation.utils import root_relative_path
+from appmap._implementation.utils import root_relative_path, scenario_filename
 
 
 class TemplateEvent(Event):  # pylint: disable=too-few-public-methods
@@ -56,3 +64,162 @@ def render(self, orig, *args, **kwargs):
         finally:
             if rec.enabled:
                 Recorder.add_event(ReturnEvent(call_event.id, time.monotonic() - start))
+
+
+NAME_MAX = 255  # true for most filesystems
+HASH_LEN = 7  # arbitrary, but git proves it's a reasonable value
+APPMAP_SUFFIX = ".appmap.json"
+
+
+def name_hash(namepart):
+    """Returns the hex digits of the sha256 of the os.fsencode()d namepart."""
+    return sha256(os.fsencode(namepart)).hexdigest()
+
+
+def write_appmap(basedir, basename, contents):
+    """Write an appmap file into basedir.
+
+    Adds APPMAP_SUFFIX to basename; shortens the name if necessary.
+    Atomically replaces existing files. Creates the basedir if required.
+    """
+
+    if len(basename) > NAME_MAX - len(APPMAP_SUFFIX):
+        part = NAME_MAX - len(APPMAP_SUFFIX) - 1 - HASH_LEN
+        basename = basename[:part] + "-" + name_hash(basename[part:])[:HASH_LEN]
+    filename = basename + APPMAP_SUFFIX
+
+    if not basedir.exists():
+        basedir.mkdir(parents=True, exist_ok=True)
+
+    with NamedTemporaryFile(mode="w", dir=basedir, delete=False) as tmp:
+        tmp.write(contents)
+    os.replace(tmp.name, basedir / filename)
+
+
+def create_appmap_file(
+    output_dir,
+    request_method,
+    request_path_info,
+    request_full_path,
+    response,
+    headers,
+    rec,
+):
+    start_time = datetime.datetime.now()
+    appmap_name = (
+        request_method
+        + " "
+        + request_path_info
+        + " ("
+        + str(response.status_code)
+        + ") - "
+        + start_time.strftime("%T.%f")[:-3]
+    )
+    appmap_basename = scenario_filename(
+        "_".join([str(start_time.timestamp()), request_full_path])
+    )
+    appmap_file_path = os.path.join(output_dir, appmap_basename)
+    metadata = {
+        "name": appmap_name,
+        "timestamp": start_time.timestamp(),
+        "recorder": {"name": "record_requests"},
+    }
+    write_appmap(output_dir, appmap_basename, generation.dump(rec, metadata))
+    headers["AppMap-Name"] = os.path.abspath(appmap_name)
+    headers["AppMap-File-Name"] = os.path.abspath(appmap_file_path) + APPMAP_SUFFIX
+
+
+class AppmapMiddleware:
+    def before_request_hook(
+        self, request, request_path, record_url, recording_is_running
+    ):
+        if request_path == record_url:
+            return None, None, None
+
+        start = None
+        call_event_id = None
+        if Env.current.enabled or recording_is_running:
+            # It should be recording or it's currently recording.  The
+            # recording is either
+            # a) remote, enabled by POST to /_appmap/record, which set
+            #    recording_is_running, or
+            # b) requests, set by Env.current.record_all_requests, or
+            # c) both remote and requests; there are multiple active recorders.
+            if not Env.current.record_all_requests and recording_is_running:
+                # a)
+                rec = Recorder()
+            else:
+                # b) or c)
+                rec = Recorder(_EventIds.get_thread_id())
+                rec.start_recording()
+                # Each time an event is added for a thread_id it's also
+                # added to the global Recorder().  So don't add the event
+                # to the global Recorder() explicitly because that would
+                # add the event in it twice.
+
+            if rec.enabled:
+                start, call_event_id = self.before_request_main(rec, request)
+
+        return rec, start, call_event_id
+
+    @abstractmethod
+    def before_request_main(self, rec):
+        raise NotImplementedError("Must override before_request_main")
+
+    def after_request_hook(
+        self,
+        request,
+        request_path,
+        record_url,
+        recording_is_running,
+        request_method,
+        request_base_url,
+        response,
+        response_headers,
+        start,
+        call_event_id,
+    ):
+        if request_path == record_url:
+            return response
+
+        if Env.current.enabled or recording_is_running:
+            # It should be recording or it's currently recording.  The
+            # recording is either
+            # a) remote, enabled by POST to /_appmap/record, which set
+            #    self.recording.is_running, or
+            # b) requests, set by Env.current.record_all_requests, or
+            # c) both remote and requests; there are multiple active recorders.
+            if not Env.current.record_all_requests and recording_is_running:
+                # a)
+                rec = Recorder()
+                if rec.enabled:
+                    self.after_request_main(rec, response, start, call_event_id)
+            else:
+                # b) or c)
+                rec = Recorder(_EventIds.get_thread_id())
+                # Each time an event is added for a thread_id it's also
+                # added to the global Recorder().  So don't add the event
+                # to the global Recorder() explicitly because that would
+                # add the event in it twice.
+                try:
+                    if rec.enabled:
+                        self.after_request_main(rec, response, start, call_event_id)
+
+                    output_dir = Env.current.output_dir / "requests"
+                    create_appmap_file(
+                        output_dir,
+                        request_method,
+                        request_path,
+                        request_base_url,
+                        response,
+                        response_headers,
+                        rec,
+                    )
+                finally:
+                    rec.stop_recording()
+
+        return response
+
+    @abstractmethod
+    def after_request_main(self, rec, response, start, call_event_id):
+        raise NotImplementedError("Must override after_request_main")