[WiP] docs: external dependency dashboard.

This patch introduces a set of automatically generated tables (based on repository_locations.bzl) that enumerate the external dependencies that feature on Envoy's data/control planes, test, build, etc. Version and CPE information is currently included. In the future, we will also have last updated, distinguish core vs. extensions and populate with external dependency process maturity information. Part of envoyproxy#10471 Signed-off-by: Harvey Tuch <[email protected]>
htuch · Aug 13, 2020 · 384f0fd · 384f0fd
1 parent 466245b
commit 384f0fd
Show file tree

Hide file tree

Showing 6 changed files with 152 additions and 10 deletions.
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -19,24 +19,36 @@ WINDOWS_SKIP_TARGETS = [
 # archives, e.g. cares.
 BUILD_ALL_CONTENT = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""
 
+def _fail_missing_attribute(attr, key):
+    fail("The '%s' attribute must be defined for external dependecy " % attr + key)
+
 # Method for verifying content of the DEPENDENCY_REPOSITORIES defined in bazel/repository_locations.bzl
 # Verification is here so that bazel/repository_locations.bzl can be loaded into other tools written in Python,
 # and as such needs to be free of bazel specific constructs.
 def _repository_locations():
     locations = dict(DEPENDENCY_REPOSITORIES)
     for key, location in locations.items():
         if "sha256" not in location or len(location["sha256"]) == 0:
-            fail("SHA256 missing for external dependency " + str(location["urls"]))
+            _fail_missing_attribute("sha256", key)
+
+        #if "project_name" not in location:
+        #    _fail_missing_attribute("project_name", key)
+
+        #if "project_url" not in location:
+        #    _fail_missing_attribute("project_url", key)
+
+        #if "version" not in location:
+        #    _fail_missing_attribute("version", key)
 
         if "use_category" not in location:
-            fail("The 'use_category' attribute must be defined for external dependecy " + str(location["urls"]))
+            _fail_missing_attribute("use_category", key)
 
         if "cpe" not in location and not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]:
-            fail("The 'cpe' attribute must be defined for external dependecy " + str(location["urls"]))
+            _fail_missing_attribute("cpe", key)
 
         for category in location["use_category"]:
             if category not in USE_CATEGORIES:
-                fail("Unknown use_category value '" + category + "' for dependecy " + str(location["urls"]))
+                fail("Unknown use_category value '" + category + "' for dependecy " + key)
 
     return locations
 

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -40,24 +40,37 @@ USE_CATEGORIES = [
 # Components with these use categories are not required to specify the 'cpe' annotation.
 USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "test", "other"]
 
+BAZEL_COMPDB_VERSION = "0.4.5"
+BAZEL_GAZELLE_VERSION = "0.19.1"
+BAZEL_TOOLCHAINS_VERSION = "3.4.0"
+
 DEPENDENCY_REPOSITORIES = dict(
     bazel_compdb = dict(
+        project_name = "bazil-compilation-database",
+        project_url = "https://github.com/grailbio/bazel-compilation-database",
+        version = BAZEL_COMPDB_VERSION,
         sha256 = "bcecfd622c4ef272fd4ba42726a52e140b961c4eac23025f18b346c968a8cfb4",
-        strip_prefix = "bazel-compilation-database-0.4.5",
-        urls = ["https://github.com/grailbio/bazel-compilation-database/archive/0.4.5.tar.gz"],
+        strip_prefix = "bazel-compilation-database-" + BAZEL_COMPDB_VERSION,
+        urls = ["https://github.com/grailbio/bazel-compilation-database/archive/" + BAZEL_COMPDB_VERSION + ".tar.gz"],
         use_category = ["build"],
     ),
     bazel_gazelle = dict(
+        project_name = "Gazelle",
+        project_url = "https://github.com/bazelbuild/bazel-gazelle",
+        version = BAZEL_GAZELLE_VERSION,
         sha256 = "86c6d481b3f7aedc1d60c1c211c6f76da282ae197c3b3160f54bd3a8f847896f",
-        urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz"],
+        urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v" + BAZEL_GAZELLE_VERSION + ".tar.gz"],
         use_category = ["build"],
     ),
     bazel_toolchains = dict(
+        project_name = "bazel-toolchains",
+        project_url = "https://github.com/bazelbuild/bazel-toolchains",
+        version = BAZEL_TOOLCHAINS_VERSION,
         sha256 = "882fecfc88d3dc528f5c5681d95d730e213e39099abff2e637688a91a9619395",
-        strip_prefix = "bazel-toolchains-3.4.0",
+        strip_prefix = "bazel-toolchains-" + BAZEL_TOOLCHAINS_VERSION,
         urls = [
-            "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.4.0/bazel-toolchains-3.4.0.tar.gz",
-            "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/3.4.0.tar.gz",
+            "https://github.com/bazelbuild/bazel-toolchains/releases/download/" + BAZEL_TOOLCHAINS_VERSION + "/bazel-toolchains-3.4.0.tar.gz",
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/" + BAZEL_TOOLCHAINS_VERSION + ".tar.gz",
         ],
         use_category = ["build"],
     ),

diff --git a/docs/build.sh b/docs/build.sh
@@ -70,6 +70,9 @@ BAZEL_BUILD_OPTIONS+=" --remote_download_outputs=all --strategy=protodoc=sandbox
 mkdir -p "${GENERATED_RST_DIR}"/intro/arch_overview/security
 ./docs/generate_extension_rst.py "${EXTENSION_DB_PATH}" "${GENERATED_RST_DIR}"/intro/arch_overview/security
 
+# Generate RST for external dependency docs in intro/arch_overview/security.
+./docs/generate_external_dep_rst.py "${GENERATED_RST_DIR}"/intro/arch_overview/security
+
 function generate_api_rst() {
   declare -r API_VERSION=$1
   echo "Generating ${API_VERSION} API RST..."

diff --git a/docs/generate_external_dep_rst.py b/docs/generate_external_dep_rst.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+# Generate RST lists of external dependencies.
+
+from collections import defaultdict, namedtuple
+import pathlib
+import sys
+import urllib.parse
+
+from importlib.util import spec_from_loader, module_from_spec
+from importlib.machinery import SourceFileLoader
+
+# bazel/repository_locations.bzl must have a .bzl suffix for Starlark import, so
+# we are forced to do this workaround.
+_repository_locations_spec = spec_from_loader(
+    'repository_locations',
+    SourceFileLoader('repository_locations', 'bazel/repository_locations.bzl'))
+repository_locations = module_from_spec(_repository_locations_spec)
+_repository_locations_spec.loader.exec_module(repository_locations)
+
+
+def CsvTable(headers, widths, rows):
+  csv_rows = '\n  '.join(', '.join(row) for row in rows)
+  return f'''.. csv-table::
+  :header: {', '.join(headers)}
+  :widths: {', '.join(str(w) for w in widths) }
+
+  {csv_rows}
+
+'''
+
+
+def RstLink(text, url):
+  return f'`{text} <{url}>`__'
+
+
+def NistCpeUrl(cpe):
+  encoded_cpe = urllib.parse.quote(cpe)
+  return 'https://nvd.nist.gov/products/cpe/search/results?keyword=%s&status=FINAL&orderBy=CPEURI&namingFormat=2.3' % encoded_cpe
+
+
+if __name__ == '__main__':
+  security_rst_root = sys.argv[1]
+
+  Dep = namedtuple('Dep', ['name', 'sort_name', 'version', 'cpe'])
+  use_categories = defaultdict(list)
+  for k, v in repository_locations.DEPENDENCY_REPOSITORIES.items():
+    cpe = v.get('cpe', '')
+    if cpe == 'N/A':
+      cpe = ''
+    if cpe:
+      cpe = RstLink(cpe, NistCpeUrl(cpe))
+    project_name = v.get('project_name', k)
+    if 'project_url' in v:
+      project_url = v['project_url']
+      name = RstLink(project_name, project_url)
+    else:
+      name = project_name
+    version = RstLink(v.get('version', '?'), v['urls'][0])
+    dep = Dep(name, project_name.lower(), version, cpe)
+    for category in v['use_category']:
+      use_categories[category].append(dep)
+
+  def CsvRow(dep):
+    return [dep.name, dep.version, dep.cpe]
+
+  for category, deps in use_categories.items():
+    output_path = pathlib.Path(security_rst_root, f'external_dep_{category}.rst')
+    content = CsvTable(['Name', 'Version', 'CPE'], [1, 1, 1],
+                       [CsvRow(dep) for dep in sorted(deps, key=lambda d: d.sort_name)])
+    output_path.write_text(content)
diff --git a/docs/root/intro/arch_overview/security/external_deps.rst b/docs/root/intro/arch_overview/security/external_deps.rst
@@ -0,0 +1,42 @@
+.. _arch_overview_external_deps:
+
+External dependencies
+=====================
+
+Below we enumerate the external dependencies that may be linked into the Envoy binary.
+
+Data plane dependencies:
+
+.. include:: external_dep_dataplane.rst
+
+Control plane dependencies:
+
+.. include:: external_dep_controlplane.rst
+
+Observability dependencies:
+
+.. include:: external_dep_observability.rst
+
+Test dependencies:
+
+.. include:: external_dep_test.rst
+
+Build dependencies:
+
+.. include:: external_dep_build.rst
+
+Miscellaneous dependencies:
+
+.. include:: external_dep_other.rst
+
+We exclude dependencies that only are used in CI or developer tooling above.
+
+TODO: also analyze `api/bazel/repository_locations.py` and `bazel/dependency_imports.bzl`.
+
+TODO: distinguish deps in core vs. extensions.
+
+TODO: add last updated column.
+
+TODO: integrate version into CPE links.
+
+TODO: populate with external dep maturity status.
diff --git a/docs/root/intro/arch_overview/security/security.rst b/docs/root/intro/arch_overview/security/security.rst
@@ -5,6 +5,7 @@ Security
   :maxdepth: 2
 
   threat_model
+  external_deps
   google_vrp
   ssl
   jwt_authn_filter