[Pal/Linux-SGX] Add OCALL profiling modes

This change extends the SGX profiler to handle OCALLs, not only
asynchronous enclave exit. Depending on settings, we either
record enclave state during OCALL, or the outer OCALL handler
being executed.

Signed-off-by: Paweł Marczewski <[email protected]>

Documentation/devel/performance.rst

@@ -533,6 +533,33 @@ measuring the value of instruction pointer on each asynchronous enclave exit
 as page faults. While we attempt to measure time (and not only count
 occurences), the results might be inaccurate.
 
+.. _sgx-profile-ocall:
+
+OCALL profiling
+"""""""""""""""
+
+It's also possible to discover what OCALLs are being executed, which should help
+attribute the EEXIT numbers given by ``sgx.enable_stats``. There are two ways to
+do that:
+
+* Use ``sgx.profile.mode = "ocall_inner"`` and ``sgx.profile.with_stack =
+  1``. This will give you a report on what enclave code is causing the OCALLs
+  (best viewed with ``perf report --no-children``).
+
+  The ``with_stack`` option is important: without it, the report will only show
+  the last function before enclave exit, which is usually the same regardless of
+  which OCALL we're executing.
+
+* Use ``sgx.profile.mode = "ocall_outer"``. This will give you a report on what
+  outer PAL code is handling the OCALLs (``sgx_ocall_open``, ``sgx_ocall_write``
+  etc.)
+
+**Warning**: The report for OCALL modes should be interpreted in term of *number
+of OCALLs*, not time spent in them. The profiler records a sample every time an
+OCALL is executed, and ``perf report`` displays percentages based on the number
+of samples.
+
+
 Other useful tools for profiling
 --------------------------------
 

Documentation/manifest-syntax.rst

@@ -551,12 +551,31 @@ collect samples and save them to ``sgx-perf-<PID>.data``.
 The saved files can be viewed with the ``perf`` tool, e.g. ``perf report -i
 sgx-perf.data``.
 
-See :doc:`devel/performance` for more information.
+See :ref:`sgx-profile` for more information.
 
 *Note:* this option is insecure and cannot be used with production enclaves
 (``sgx.debug = 0``). If the production enclave is started with this option set,
 Graphene will fail initialization of the enclave.
 
+::
+
+    sgx.profile.mode = ["aex"|"ocall_inner"|"ocall_outer"]
+    (Default: "aex")
+
+Specifies what events to record:
+
+* ``aex``: Records enclave state during asynchronous enclave exit (AEX). Use
+  this to check where the CPU time is spent in the enclave.
+
+* ``ocall_inner``: Records enclave state during OCALL.
+
+* ``ocall_outer``: Records the outer OCALL function, i.e. what OCALL handlers
+  are going to be executed. Does not include stack information (cannot be used
+  with ``sgx.profile.with_stack = 1``).
+
+See also :ref:`sgx-profile-ocall` for more detailed advice regarding the OCALL
+modes.
+
 ::
 
     sgx.profile.with_stack = [1|0]
@@ -577,3 +596,6 @@ lower overhead.
 
 Note that the accuracy is limited by how often the process is interrupted by
 Linux scheduler: the effective maximum is 250 samples per second.
+
+**Note**: This option applies only to ``aex`` mode. In the ``ocall_*`` modes,
+currently all samples are taken.

Pal/src/host/Linux-SGX/Makefile

@@ -78,6 +78,7 @@ urts-objs = \
 	sgx_platform.o \
 	sgx_process.o \
 	sgx_profile.o \
+	sgx_profile_glibc.o \
 	sgx_gdb_info.o \
 	sgx_thread.o \
 	quote/aesm.pb-c.o \

Pal/src/host/Linux-SGX/enclave_entry.S

@@ -631,13 +631,16 @@ sgx_ocall:
 	# %rax is argument to EEXIT
 	# %rbx is argument to EEXIT
 	# %rcx is set to AEP by EEXIT
-	xorq %rdx, %rdx
 	# %rsi, %rdi are arguments to the untrusted code
 
 #ifdef DEBUG
+	# Store pointer to context in RDX, for the SGX profiler.
+	movq %gs:SGX_PRE_OCALL_STACK, %rdx
+
 	# Keep callee-saved registers in order to recover stack later (see __morestack() below).
 #else
 	# In non-debug mode, clear these registers to prevent information leaks.
+	xorq %rdx, %rdx
 	xorq %rbp, %rbp
 	xorq %r12, %r12
 	xorq %r13, %r13

Pal/src/host/Linux-SGX/sgx_entry.S

@@ -84,9 +84,9 @@ async_exit_pointer:
 	.cfi_def_cfa_register %rbp
 	andq $~0xF, %rsp
 
-	# Call sgx_profile_sample with %rdi = TCS
+	# Call sgx_profile_sample_aex with %rdi = TCS
 	movq %rbx, %rdi
-	call sgx_profile_sample
+	call sgx_profile_sample_aex
 
 	# Restore stack
 	movq %rbp, %rsp
@@ -139,7 +139,26 @@ sgx_raise:
 	movq %rsp, %rbp
 	.cfi_offset %rbp, -16
 	.cfi_def_cfa_register %rbp
+
+#if DEBUG
+    # Adjust stack and save RDI
+	subq $8, %rsp
 	andq $~0xF, %rsp  # Required by System V AMD64 ABI.
+	movq %rdi, -8(%rbp)
+
+	# Call sgx_profile_sample_ocall_outer with RBX (ocall handler)
+	movq %rbx, %rdi
+	call sgx_profile_sample_ocall_outer
+
+	# Call sgx_profile_sample_ocall_inner with RDX (pointer to in-enclave context)
+	movq %rdx, %rdi
+	call sgx_profile_sample_ocall_inner
+
+	# Restore RDI
+	movq -8(%rbp), %rdi
+#else
+	andq $~0xF, %rsp  # Required by System V AMD64 ABI.
+#endif
 
 	callq *%rbx
 

Pal/src/host/Linux-SGX/sgx_internal.h

@@ -73,6 +73,7 @@ struct pal_enclave {
 #ifdef DEBUG
     /* profiling */
     bool profile_enable;
+    int profile_mode;
     char profile_filename[64];
     bool profile_with_stack;
     int profile_frequency;
@@ -164,6 +165,12 @@ void update_debugger(void);
 #define SGX_PROFILE_DEFAULT_FREQUENCY 50
 #define SGX_PROFILE_MAX_FREQUENCY 250
 
+enum {
+    SGX_PROFILE_MODE_AEX = 1,
+    SGX_PROFILE_MODE_OCALL_INNER = 2,
+    SGX_PROFILE_MODE_OCALL_OUTER = 3,
+};
+
 /* Filenames for saved data */
 #define SGX_PROFILE_FILENAME "sgx-perf.data"
 #define SGX_PROFILE_FILENAME_WITH_PID "sgx-perf-%d.data"
@@ -174,11 +181,20 @@ int sgx_profile_init(void);
 /* Finalize and close file */
 void sgx_profile_finish(void);
 
-/* Record a sample */
-void sgx_profile_sample(void* tcs);
+/* Record a sample during AEX */
+void sgx_profile_sample_aex(void* tcs);
+
+/* Record a sample during OCALL (inner state) */
+void sgx_profile_sample_ocall_inner(void* enclave_gpr);
+
+/* Record a sample during OCALL (function to be executed) */
+void sgx_profile_sample_ocall_outer(void* ocall_func);
 
 /* Record a new mapped ELF */
 void sgx_profile_report_elf(const char* filename, void* addr);
+
+/* Record all ELFs from outer PAL */
+void sgx_profile_report_urts_elfs(void);
 #endif
 
 /* perf.data output (sgx_perf_data.h) */

Pal/src/host/Linux-SGX/sgx_main.c

@@ -534,6 +534,9 @@ static int initialize_enclave(struct pal_enclave* enclave, const char* manifest_
 
     debug_map_add(enclave->libpal_uri + URI_PREFIX_FILE_LEN, (void*)pal_area->addr);
     sgx_profile_report_elf(enclave->libpal_uri + URI_PREFIX_FILE_LEN, (void*)pal_area->addr);
+
+    /* Report outer PAL maps to profiler, so that we can record samples pointing to outer PAL. */
+    sgx_profile_report_urts_elfs();
 #endif
 
     ret = 0;
@@ -765,6 +768,29 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info)
         goto out;
     }
 
+    char* profile_mode_str = NULL;
+    ret = toml_string_in(manifest_root, "sgx.profile.mode", &profile_mode_str);
+    if (ret < 0) {
+        urts_log_error("Cannot parse 'sgx.profile.mode' "
+                       "(the value must be \"aex\", \"ocall_inner\" or \"ocall_outer\")\n");
+        ret = -EINVAL;
+        goto out;
+    }
+    if (!profile_mode_str) {
+        enclave_info->profile_mode = SGX_PROFILE_MODE_AEX;
+    } else if (!strcmp(profile_mode_str, "aex")) {
+        enclave_info->profile_mode = SGX_PROFILE_MODE_AEX;
+    } else if (!strcmp(profile_mode_str, "ocall_inner")) {
+        enclave_info->profile_mode = SGX_PROFILE_MODE_OCALL_INNER;
+    } else if (!strcmp(profile_mode_str, "ocall_outer")) {
+        enclave_info->profile_mode = SGX_PROFILE_MODE_OCALL_OUTER;
+    } else {
+        urts_log_error("Invalid 'sgx.profile.mode' "
+                       "(the value must be \"aex\", \"ocall_inner\" or \"ocall_outer\")\n");
+        ret = -EINVAL;
+        goto out;
+    }
+
     int64_t profile_with_stack;
     ret = toml_int_in(manifest_root, "sgx.profile.with_stack", /*defaultval=*/0,
                       &profile_with_stack);
@@ -775,6 +801,15 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info)
     }
     enclave_info->profile_with_stack = profile_with_stack;
 
+    if (enclave_info->profile_with_stack &&
+        enclave_info->profile_mode == SGX_PROFILE_MODE_OCALL_OUTER) {
+
+        urts_log_error("Invalid 'sgx.profile.mode' and 'sgx.profile.with_stack' combination "
+                       "(\"ocall_outer\" mode cannot be used with stack)\n");
+        ret = -EINVAL;
+        goto out;
+    }
+
     int64_t profile_frequency;
     ret = toml_int_in(manifest_root, "sgx.profile.frequency", SGX_PROFILE_DEFAULT_FREQUENCY,
                       &profile_frequency);