Converting envoy configs to V2

bazel/external/pyyaml.BUILD

@@ -0,0 +1,6 @@
+py_library(
+    name = "yaml",
+    srcs = glob(["lib/yaml/**/*.py"]),
+    imports = ["lib"],
+    visibility = ["//visibility:public"],
+)

bazel/repositories.bzl

@@ -141,6 +141,15 @@ def _python_deps():
         name = "jinja2",
         actual = "@com_github_pallets_jinja//:jinja2",
     )
+    _repository_impl(
+        name = "com_github_yaml_pyyaml",
+        build_file = "@envoy//bazel/external:pyyaml.BUILD",
+    )
+    native.bind(
+        name = "yaml",
+        actual = "@com_github_yaml_pyyaml//:yaml",
+    )
+
 
 # Bazel native C++ dependencies. For the depedencies that doesn't provide autoconf/automake builds.
 def _cc_deps():

bazel/repository_locations.bzl

@@ -80,6 +80,10 @@ REPOSITORY_LOCATIONS = dict(
         commit = "f54b0e47a08782a6131cc3d60f94d038fa6e0a51",  # v1.1.0
         remote = "https://github.com/tencent/rapidjson",
     ),
+    com_github_yaml_pyyaml = dict(
+        commit = "b6cbfeec35e019734263a8f4e6a3340e94fe0a4f",
+        remote = "https://github.com/yaml/pyyaml",
+    ),
     com_google_googletest = dict(
         commit = "43863938377a9ea1399c0596269e0890b5c5515a",
         remote = "https://github.com/google/googletest",

configs/BUILD

@@ -13,10 +13,13 @@ envoy_py_test_binary(
     name = "configgen",
     srcs = ["configgen.py"],
     data = glob([
-        "*.json",
         "*.yaml",
+        "*.json",
     ]),
-    external_deps = ["jinja2"],
+    external_deps = [
+        "jinja2",
+        "yaml",
+    ],
 )
 
 filegroup(

configs/access_log_format_helper_v2.template.yaml

@@ -0,0 +1,15 @@
+{% macro ingress_sampled_log() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
+{% endmacro %}
+
+{% macro ingress_full() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
+{% endmacro %}
+
+{% macro egress_error_log() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"
+{% endmacro %}
+
+{% macro egress_error_amazon_service() -%}
+  format: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%RESP(X-AMZN-RequestId)%\"\n"
+{% endmacro %}

configs/configgen.py

@@ -49,11 +49,15 @@
 external_virtual_hosts = [
 {
     'name': 'dynamodb_iad',
-    'address': "tcp://127.0.0.1:9204",
+    'address': "127.0.0.1",
+    'protocol':"TCP",
+    'port_value':"9204",
     'hosts': [
         {
             'name': 'dynamodb_iad', 'domain': '*',
-            'remote_address': 'dynamodb.us-east-1.amazonaws.com:443',
+            'remote_address': 'dynamodb.us-east-1.amazonaws.com',
+            'protocol': 'TCP',
+            'port_value': '443',
             'verify_subject_alt_name': [ 'dynamodb.us-east-1.amazonaws.com' ],
             'ssl': True
         }
@@ -69,12 +73,14 @@
 # as it demonstrates how to setup TCP proxy and the network rate limit filter.
 mongos_servers = {
     'somedb': {
-        'address': "tcp://127.0.0.1:27019",
+        'address': "127.0.0.1",
+        'protocol': "TCP",
+        'port_value': 27019,
         'hosts': [
-            "router1.yourcompany.net:27817",
-            "router2.yourcompany.net:27817",
-            "router3.yourcompany.net:27817",
-            "router4.yourcompany.net:27817",
+            {'port_value' : 27817, 'address':'router1.yourcompany.net' , 'protocol': 'TCP'},
+            {'port_value' : 27817, 'address':'router2.yourcompany.net' , 'protocol': 'TCP'},
+            {'port_value' : 27817, 'address':'router3.yourcompany.net' , 'protocol': 'TCP'},
+            {'port_value' : 27817, 'address':'router4.yourcompany.net' , 'protocol': 'TCP'},
         ],
         'ratelimit': True
     }
@@ -85,20 +91,18 @@ def generate_config(template_path, template, output_file, **context):
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path, followlinks=True),
                              undefined=jinja2.StrictUndefined)
     raw_output = env.get_template(template).render(**context)
-    # Verify valid JSON and then dump it nicely formatted to avoid jinja pain.
-    output = json.loads(raw_output, object_pairs_hook=OrderedDict)
     with open(output_file, 'w') as fh:
-        json.dump(output, fh, indent=2)
+        fh.write(raw_output)
 
 # Generate a demo config for the main front proxy. This sets up both HTTP and HTTPS listeners,
 # as well as a listener for the double proxy to connect to via SSL client authentication.
-generate_config(SCRIPT_DIR, 'envoy_front_proxy.template.json',
-                '{}/envoy_front_proxy.json'.format(OUT_DIR), clusters=front_envoy_clusters)
+generate_config(SCRIPT_DIR, 'envoy_front_proxy_v2.template.yaml',
+                '{}/envoy_front_proxy.v2.yaml'.format(OUT_DIR), clusters=front_envoy_clusters)
 
 # Generate a demo config for the double proxy. This sets up both an HTTP and HTTPS listeners,
 # and backhauls the traffic to the main front proxy.
-generate_config(SCRIPT_DIR, 'envoy_double_proxy.template.json',
-                '{}/envoy_double_proxy.json'.format(OUT_DIR))
+generate_config(SCRIPT_DIR, 'envoy_double_proxy_v2.template.yaml',
+                '{}/envoy_double_proxy.v2.yaml'.format(OUT_DIR))
 
 # Generate a demo config for the service to service (local) proxy. This sets up several different
 # listeners:
@@ -108,11 +112,11 @@ def generate_config(template_path, template, output_file, **context):
 # optional external service ports: built from external_virtual_hosts above. Each external host
 #                                  that Envoy proxies to listens on its own port.
 # optional mongo ports: built from mongos_servers above.
-generate_config(SCRIPT_DIR, 'envoy_service_to_service.template.json',
-                '{}/envoy_service_to_service.json'.format(OUT_DIR),
+generate_config(SCRIPT_DIR, 'envoy_service_to_service_v2.template.yaml',
+                '{}/envoy_service_to_service.yaml'.format(OUT_DIR),
                 internal_virtual_hosts=service_to_service_envoy_clusters,
                 external_virtual_hosts=external_virtual_hosts,
                 mongos_servers=mongos_servers)
 
-for google_ext in ['json', 'yaml', 'v2.yaml']:
+for google_ext in ['v2.yaml']:
   shutil.copy(os.path.join(SCRIPT_DIR, 'google_com_proxy.%s' % google_ext), OUT_DIR)