Move from toml library to tomli + tomli-w

`toml` Python library (for parsing and dumping TOML files) is buggy and
doesn't support the full spec of TOML 1.0.0. This commit replaces it
with more robust `tomli` (for parsing) and `tomli-w` (for dumping).

Note that GSC supports both newer versions of Gramine (that also use
`tomli`) and older versions of Gramine (that use `toml`).

Signed-off-by: Dmitrii Kuvaiskii <[email protected]>

Documentation/index.rst

@@ -42,7 +42,8 @@ pip. GSC requires Python 3.6 or later.
 .. code-block:: sh
 
    sudo apt-get install docker.io python3 python3-pip
-   pip3 install docker jinja2 toml pyyaml
+   pip3 install docker jinja2 tomli tomli-w pyyaml
+   pip3 install toml  # for compatibility with Gramine v1.3 or lower
 
 SGX software stack
 ------------------

finalize_manifest.py

@@ -11,7 +11,8 @@
 import sys
 
 import jinja2
-import toml
+import tomli
+import tomli_w
 
 def is_utf8(filename_bytes):
     try:
@@ -75,7 +76,9 @@ def generate_trusted_files(root_dir, already_added_files):
                 # Python TOML parser has a bug that prevents it from correctly handling `\x`
                 # sequences in strings (see https://github.com/uiri/toml/issues/404). Fortunately,
                 # the only files that exhibit such pattern are systemd helper files which graminized
-                # apps will never access anyway. FIXME: Switch to another, less buggy TOML parser.
+                # apps will never access anyway.
+                # FIXME: Now we switched to `tomli`, but GSC can still use Gramine v1.3 or lower
+                #        which uses `toml`. When GSC removes support for v1.3, can remove this.
                 print(f'\t[from inside Docker container] File {filename} contains `\\x` sequence '
                        'and will be skipped from `sgx.trusted_files`!')
                 continue
@@ -121,7 +124,7 @@ def main(args=None):
 
     manifest = '/gramine/app_files/entrypoint.manifest'
     rendered_manifest = env.get_template(manifest).render()
-    rendered_manifest_dict = toml.loads(rendered_manifest)
+    rendered_manifest_dict = tomli.loads(rendered_manifest)
     already_added_files = extract_files_from_user_manifest(rendered_manifest_dict)
 
     if 'allow_all_but_log' not in rendered_manifest_dict['sgx'].get('file_check_policy', ''):
@@ -130,8 +133,8 @@ def main(args=None):
     else:
         print(f'\t[from inside Docker container] Skipping trusted files generation. This image must not be used in production.')
 
-    with open(manifest, 'w') as manifest_file:
-        toml.dump(rendered_manifest_dict, manifest_file)
+    with open(manifest, 'wb') as manifest_file:
+        tomli_w.dump(rendered_manifest_dict, manifest_file)
     print(f'\t[from inside Docker container] Successfully finalized `{manifest}`.')
 
 if __name__ == '__main__':

gsc.py

@@ -16,7 +16,8 @@
 
 import docker  # pylint: disable=import-error
 import jinja2
-import toml    # pylint: disable=import-error
+import tomli   # pylint: disable=import-error
+import tomli_w # pylint: disable=import-error
 import yaml    # pylint: disable=import-error
 
 def gsc_image_name(original_image_name):
@@ -214,18 +215,18 @@ def gsc_build(args):
     #   - base Docker image's environment variables
     #   - additional, user-provided manifest options
     entrypoint_manifest_render = env.get_template(f'{distro}/entrypoint.manifest.template').render()
-    entrypoint_manifest_dict = toml.loads(entrypoint_manifest_render)
+    entrypoint_manifest_dict = tomli.loads(entrypoint_manifest_render)
 
     base_image_environment = extract_environment_from_image_config(original_image.attrs['Config'])
-    base_image_dict = toml.loads(base_image_environment)
+    base_image_dict = tomli.loads(base_image_environment)
 
     user_manifest_contents = ''
     if not os.path.exists(args.manifest):
         raise FileNotFoundError(f'Manifest file {args.manifest} does not exist')
     with open(args.manifest, 'r') as user_manifest_file:
         user_manifest_contents = user_manifest_file.read()
 
-    user_manifest_dict = toml.loads(user_manifest_contents)
+    user_manifest_dict = tomli.loads(user_manifest_contents)
 
     # Support deprecated syntax: replace old-style TOML-dict (`sgx.trusted_files.key = "file:foo"`)
     # with new-style TOML-array (`sgx.trusted_files = ["file:foo"]`) in the user manifest
@@ -246,8 +247,8 @@ def gsc_build(args):
     merged_manifest_dict = merge_two_dicts(user_manifest_dict, entrypoint_manifest_dict)
     merged_manifest_dict = merge_two_dicts(merged_manifest_dict, base_image_dict)
 
-    with open(tmp_build_path / 'entrypoint.manifest', 'w') as entrypoint_manifest:
-        toml.dump(merged_manifest_dict, entrypoint_manifest)
+    with open(tmp_build_path / 'entrypoint.manifest', 'wb') as entrypoint_manifest:
+        tomli_w.dump(merged_manifest_dict, entrypoint_manifest)
 
     # copy helper script to finalize the manifest from within graminized Docker image
     shutil.copyfile('finalize_manifest.py', tmp_build_path / 'finalize_manifest.py')
@@ -447,7 +448,7 @@ def gsc_info_image(args):
             print(f'Could not extract Intel SGX-related information from image {args.image}.')
             sys.exit(1)
 
-        print(toml.dumps(sigstruct))
+        print(tomli_w.dumps(sigstruct))
 
 
 argparser = argparse.ArgumentParser()

templates/centos/Dockerfile.build.template

@@ -16,7 +16,11 @@ RUN dnf update -y \
         python3-cryptography \
         python3-pip \
         python3-protobuf \
-    && /usr/bin/python3 -B -m pip install click jinja2 protobuf 'toml>=0.10'
+    && /usr/bin/python3 -B -m pip install click jinja2 protobuf \
+                                          'tomli>=1.1.0' 'tomli-w>=0.4.0'
+
+# For compatibility with Gramine v1.3 or lower
+RUN /usr/bin/python3 -B -m pip install 'toml>=0.10'
 
 # Install pyelftools after the installation of epel-release as it is provided by the EPEL repo
 RUN dnf install -y python3-pyelftools

templates/centos/Dockerfile.compile.template

@@ -36,5 +36,9 @@ RUN dnf update -y \
         python3-protobuf \
         rpm-build \
         wget \
-    && /usr/bin/python3 -B -m pip install 'toml>=0.10' 'meson>=0.56'
+    && /usr/bin/python3 -B -m pip install 'tomli>=1.1.0' 'tomli-w>=0.4.0' 'meson>=0.56'
+
+# For compatibility with Gramine v1.3 or lower
+RUN /usr/bin/python3 -B -m pip install 'toml>=0.10'
+
 {% endblock %}

templates/debian/Dockerfile.build.template

@@ -15,7 +15,11 @@ RUN apt-get update \
         python3-pip \
         python3-protobuf \
         python3-pyelftools \
-    && /usr/bin/python3 -B -m pip install click jinja2 protobuf 'toml>=0.10'
+    && /usr/bin/python3 -B -m pip install click jinja2 protobuf \
+                                          'tomli>=1.1.0' 'tomli-w>=0.4.0'
+
+# For compatibility with Gramine v1.3 or lower
+RUN /usr/bin/python3 -B -m pip install 'toml>=0.10'
 
 {% if debug %}
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install -y \

templates/debian/Dockerfile.compile.template

@@ -23,7 +23,10 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get update \
         python3-pip \
         python3-protobuf \
         wget \
-    && /usr/bin/python3 -B -m pip install 'toml>=0.10' 'meson>=0.56'
+    && /usr/bin/python3 -B -m pip install 'tomli>=1.1.0' 'tomli-w>=0.4.0' 'meson>=0.56'
+
+# For compatibility with Gramine v1.3 or lower
+RUN /usr/bin/python3 -B -m pip install 'toml>=0.10'
 
 COPY intel-sgx-deb.key /
 

test/generic.manifest

@@ -4,6 +4,9 @@ loader.pal_internal_mem_size = "128M"
 sgx.enclave_size = "4G"
 sgx.thread_num = 8
 
+# FIXME: `toml` Python lib that was used in Gramine v1.3 and lower hangs/fails on non-homogeneous
+#        TOML arrays, so until we fully deprecate support for Gramine v1.3 in GSC, specify items in
+#        the sgx.trusted_files array only as strings (never as dicts `{ "uri": "file:file1" }`)
 sgx.trusted_files = [
   "file:/gramine/app_files/entrypoint.manifest",  # unused entry, only to test merging of manifests
 ]