agents.yaml

# THIS FILE IS AUTO-GENERATED DO NOT EDIT
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    k8s-app: stackdriver-heapster
    version: v1.6.1
  name: heapster
  namespace: stackdriver-agents
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: stackdriver-heapster
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      creationTimestamp: null
      labels:
        k8s-app: stackdriver-heapster
        version: v1.6.1
    spec:
      containers:
      - env:
        - name: CLUSTER_NAME
          valueFrom:
            configMapKeyRef:
              name: cluster-config
              key: cluster_name
        - name: CLUSTER_LOCATION
          valueFrom:
            configMapKeyRef:
              name: cluster-config
              key: cluster_location
        - name: GOOGLE_APPLICATION_CREDENTIALS
          valueFrom:
            configMapKeyRef:
              name: google-cloud-config
              key: credentials_path
        command:
        - /heapster
        - --source=kubernetes.summary_api:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true
        - --sink=stackdriver:?cluster_name=$(CLUSTER_NAME)&cluster_location=$(CLUSTER_LOCATION)&zone=$(CLUSTER_LOCATION)&use_old_resources=false&use_new_resources=true&min_interval_sec=100&batch_export_timeout_sec=110
        image: gcr.io/stackdriver-agents/heapster-amd64:v1.6.1
        imagePullPolicy: Always
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 8082
            scheme: HTTP
          initialDelaySeconds: 180
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        name: heapster
        resources:
          limits:
            cpu: 88m
            memory: 204Mi
          requests:
            cpu: 88m
            memory: 204Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/google-cloud/
          name: google-cloud-config
      - command:
        - /pod_nanny
        - --cpu=80m
        - --extra-cpu=0.5m
        - --memory=140Mi
        - --extra-memory=4Mi
        - --threshold=5
        - --deployment=heapster
        - --container=heapster
        - --poll-period=300000
        - --estimator=exponential
        env:
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: MY_POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        image: gcr.io/google_containers/addon-resizer:1.7
        imagePullPolicy: IfNotPresent
        name: heapster-nanny
        resources:
          limits:
            cpu: 50m
            memory: 112360Ki
          requests:
            cpu: 50m
            memory: 112360Ki
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: heapster
      serviceAccountName: heapster
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: google-cloud-config
        name: google-cloud-config

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    app: stackdriver-logging-agent
  name: stackdriver-logging-agent
  namespace: stackdriver-agents
spec:
  selector:
    matchLabels:
      app: stackdriver-logging-agent
  template:
    metadata:
      labels:
        app: stackdriver-logging-agent
    spec:
      containers:
      - env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: K8S_NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: GOOGLE_APPLICATION_CREDENTIALS
          valueFrom:
            configMapKeyRef:
              name: google-cloud-config
              key: credentials_path
        - name: CLUSTER_NAME
          valueFrom:
            configMapKeyRef:
              name: cluster-config
              key: cluster_name
        - name: CLUSTER_LOCATION
          valueFrom:
            configMapKeyRef:
              name: cluster-config
              key: cluster_location
        image: gcr.io/stackdriver-agents/stackdriver-logging-agent:1.10.3
        imagePullPolicy: IfNotPresent
        livenessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - |
              LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900}; if [ ! -e /var/run/google-fluentd/buffers ]; then
                exit 1;
              fi; touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck; if [[ -z "$(find /var/run/google-fluentd/buffers -type f -newer /tmp/marker-stuck -print -quit)" ]]; then
                rm -rf /var/run/google-fluentd/buffers;
                exit 1;
              fi; touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness; if [[ -z "$(find /var/run/google-fluentd/buffers -type f -newer /tmp/marker-liveness -print -quit)" ]]; then
                exit 1;
              fi;
          failureThreshold: 3
          initialDelaySeconds: 600
          periodSeconds: 60
          successThreshold: 1
          timeoutSeconds: 1
        name: logging-agent
        resources:
          limits:
            cpu: "1"
            memory: 300Mi
          requests:
            cpu: 100m
            memory: 200Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/run
          name: varrun
        - mountPath: /var/log
          name: varlog
        - mountPath: /var/lib/docker/containers
          name: varlibdockercontainers
          readOnly: true
        - mountPath: /etc/google-fluentd/google-fluentd.conf
          subPath: google-fluentd.conf
          name: output-config-volume
        - mountPath: /etc/google-fluentd/config.d
          name: input-config-volume
        - mountPath: /etc/google-cloud/
          name: google-cloud-config
      serviceAccount: logging-agent
      serviceAccountName: logging-agent
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      tolerations:
      - operator: "Exists"
        effect: "NoExecute"
      - operator: "Exists"
        effect: "NoSchedule"
      volumes:
      - hostPath:
          path: /var/run
          type: ""
        name: varrun
      - hostPath:
          path: /var/log
          type: ""
        name: varlog
      - hostPath:
          path: /var/lib/docker/containers
          type: ""
        name: varlibdockercontainers
      - configMap:
          defaultMode: 420
          name: logging-agent-output-config
        name: output-config-volume
      - configMap:
          defaultMode: 420
          name: logging-agent-input-config
        name: input-config-volume
      - configMap:
          defaultMode: 420
          name: google-cloud-config
        name: google-cloud-config
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
---
# Config map for Logging Agent input and corresponding filter plugins.
apiVersion: v1
kind: ConfigMap
metadata:
  name: logging-agent-input-config
  namespace: stackdriver-agents
data:
  1.containers.input.conf: |-
    # This configuration file for Fluentd is used
    # to watch changes to Docker log files that live in the
    # directory /var/lib/docker/containers/ and are symbolically
    # linked to from the /var/log/containers directory using names that capture the
    # pod name and container name. These logs are then submitted to
    # Google Cloud Logging which assumes the installation of the cloud-logging plug-in.
    #
    # Example
    # =======
    # A line in the Docker log file might look like this JSON:
    #
    # {"log":"2014/09/25 21:15:03 Got request with path wombat\\n",
    #  "stream":"stderr",
    #   "time":"2014-09-25T21:15:03.499185026Z"}
    #
    # The original tag is derived from the log file's location.
    # For example a Docker container's logs might be in the directory:
    #  /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
    # and in the file:
    #  997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
    # where 997599971ee6... is the Docker ID of the running container.
    # The Kubernetes kubelet makes a symbolic link to this file on the host
    # machine in the /var/log/containers directory which includes the pod name,
    # the namespace name and the Kubernetes container name:
    #    synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
    #    ->
    #    /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
    # The /var/log directory on the host is mapped to the /var/log directory in the container
    # running this instance of Fluentd and we end up collecting the file:
    #   /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
    # This results in the tag:
    #  var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
    # where 'synthetic-logger-0.25lps-pod' is the pod name, 'default' is the
    # namespace name, 'synth-lgr' is the container name and '997599971ee6..' is
    # the container ID.
    # The record reformer is used to extract pod_name, namespace_name and
    # container_name from the tag and set them in a local_resource_id in the
    # format of:
    # 'k8s_container.<NAMESPACE_NAME>.<POD_NAME>.<CONTAINER_NAME>'.
    # The reformer also changes the tags to 'stderr' or 'stdout' based on the
    # value of 'stream'.
    # local_resource_id is later used by google_cloud plugin to determine the
    # monitored resource to ingest logs against.

    # Json Log Example:
    # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
    # CRI Log Example:
    # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
    <source>
      @type tail
      path /var/log/containers/*.log
      pos_file /var/run/google-fluentd/pos-files/gcp-containers.pos
      # Tags at this point are in the format of:
      # reform.var.log.containers.<POD_NAME>_<NAMESPACE_NAME>_<CONTAINER_NAME>-<CONTAINER_ID>.log
      tag reform.*
      read_from_head true
      <parse>
        @type multi_format
        <pattern>
          format json
          time_key time
          time_format %Y-%m-%dT%H:%M:%S.%NZ
        </pattern>
        <pattern>
          format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
          time_format %Y-%m-%dT%H:%M:%S.%N%:z
        </pattern>
      </parse>
    </source>

    <filter reform.**>
      @type parser
      format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<log>.*)/
      reserve_data true
      suppress_parse_error_log true
      emit_invalid_record_to_error false
      key_name log
    </filter>

    <filter reform.**>
      # This plugin uses environment variables KUBERNETES_SERVICE_HOST and
      # KUBERNETES_SERVICE_PORT to talk to the API server. These environment
      # variables are added by kubelet automatically.
      @type kubernetes_metadata
      # Interval in seconds to dump cache stats locally in the Fluentd log.
      stats_interval 300
      # TTL in seconds of each cached element.
      cache_ttl 30
      # Skip fetching unused metadata.
      skip_container_metadata true
      skip_master_url true
      skip_namespace_metadata true
    </filter>

    <filter reform.**>
      # We have to use record_modifier because only this plugin supports complex
      # logic to modify record the way we need.
      @type record_modifier
      enable_ruby true
      <record>
        # Extract "kubernetes"->"labels" and set them as
        # "logging.googleapis.com/labels". Prefix these labels with
        # "k8s-pod" to distinguish with other labels and avoid
        # label name collision with other types of labels.
        _dummy_ ${if record.is_a?(Hash) && record.has_key?('kubernetes') && record['kubernetes'].has_key?('labels') && record['kubernetes']['labels'].is_a?(Hash); then; record["logging.googleapis.com/labels"] = record['kubernetes']['labels'].map{ |k, v| ["k8s-pod/#{k}", v]}.to_h; end; nil}
      </record>
      # Delete this dummy field and the rest of "kubernetes" and "docker".
      remove_keys _dummy_,kubernetes,docker
    </filter>

    <match reform.**>
      @type record_reformer
      enable_ruby true
      <record>
        # Extract local_resource_id from tag for 'k8s_container' monitored
        # resource. The format is:
        # 'k8s_container.<namespace_name>.<pod_name>.<container_name>'.
        "logging.googleapis.com/local_resource_id" ${"k8s_container.#{tag_suffix[4].rpartition('.')[0].split('_')[1]}.#{tag_suffix[4].rpartition('.')[0].split('_')[0]}.#{tag_suffix[4].rpartition('.')[0].split('_')[2].rpartition('-')[0]}"}
        # Rename the field 'log' to a more generic field 'message'. This way the
        # fluent-plugin-google-cloud knows to flatten the field as textPayload
        # instead of jsonPayload after extracting 'time', 'severity' and
        # 'stream' from the record.
        message ${record['log']}
        # If 'severity' is not set, assume stderr is ERROR and stdout is INFO.
        severity ${record['severity'] || if record['stream'] == 'stderr' then 'ERROR' else 'INFO' end}
      </record>
      tag ${if record['stream'] == 'stderr' then 'raw.stderr' else 'raw.stdout' end}
      remove_keys stream,log
    </match>

    # Detect exceptions in the log output and forward them as one log entry.
    <match {raw.stderr,raw.stdout}>
      @type detect_exceptions

      remove_tag_prefix raw
      message message
      stream "logging.googleapis.com/local_resource_id"
      multiline_flush_interval 5
      max_bytes 500000
      max_lines 1000
    </match>
  2.pods.input.conf: |-
    # This configuration file for Fluentd is used
    # to watch changes to Kubernetes pod log files live in the
    # directory /var/log/pods/NAMESPACE_NAME_UID. The file name
    # is used to capture the pod namespace, name and uid. These
    # logs are then submitted to Google Cloud Logging with a
    # local_resource_id 'k8s_pod.<NAMESPACE_NAME>.<POD_NAME>'
    # which assumes the installation of the cloud-logging plug-in.
    <source>
      @type tail
      path /var/log/pods/*/*.log
      pos_file /var/run/google-fluentd/pos-files/gcp-pods.pos
      # Tags at this point are in the format of:
      # pods.reform.var.log.pods.<POD_NAMESPACE>_<POD_NAME>_<POD_UID>.<FILE_NAME>.log
      tag pods.reform.*
      read_from_head true
      <parse>
        @type none
      </parse>
    </source>
    <match pods.reform.**>
      @type record_reformer
      enable_ruby true
      <record>
        # Extract local_resource_id from tag for 'k8s_pod' monitored
        # resource. The format is:
        # 'k8s_pod.<namespace_name>.<pod_name>'.
        "logging.googleapis.com/local_resource_id" ${"k8s_pod.#{tag_suffix[5].rpartition('.')[0].split('_')[0]}.#{tag_suffix[5].rpartition('.')[0].split('_')[1]}"}
      </record>
      # Use the log file name as the tag. Currently only `gvisor` log is supported.
      tag ${"#{tag_suffix[5].rpartition('.')[0].rpartition('.')[2]}"}
    </match>
  7.system.input.conf: |-
    # Example:
    # Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
    <source>
      @type tail
      format syslog
      path /var/log/startupscript.log
      pos_file /var/run/google-fluentd/pos-files/gcp-startupscript.pos
      tag startupscript
    </source>

    # Example:
    # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
    <source>
      @type tail
      format multiline
      multiline_flush_interval 5s
      format_firstline /^\w\d{4}/
      format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
      time_format %m%d %H:%M:%S.%N
      path /var/log/kube-proxy.log
      pos_file /var/run/google-fluentd/pos-files/gcp-kube-proxy.pos
      tag kube-proxy
    </source>

    # Logs from systemd-journal for interesting services.
    # TODO(random-liu): Keep this for compatibility, remove this after
    # cri container runtime rolls out.
    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "docker.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-docker.pos
      </storage>
      read_from_head true
      tag docker
    </source>

    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "containerd.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-container-runtime.pos
      </storage>
      read_from_head true
      tag container-runtime
    </source>

    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-kubelet.pos
      </storage>
      read_from_head true
      tag kubelet
    </source>

    # kube-node-installation, kube-node-configuration, and kube-logrotate are
    # oneshots, but it's extremely valuable to have their logs on Stackdriver
    # as they can diagnose critical issues with node startup.
    # See http://cs/cloud-gke-kubernetes/cluster/gce/gci/node.yaml.
    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "kube-node-installation.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-kube-node-installation.pos
      </storage>
      read_from_head true
      tag kube-node-installation
    </source>

    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "kube-node-configuration.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-kube-node-configuration.pos
      </storage>
      read_from_head true
      tag kube-node-configuration
    </source>

    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "kube-logrotate.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-kube-logrotate.pos
      </storage>
      read_from_head true
      tag kube-logrotate
    </source>

    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-node-problem-detector.pos
      </storage>
      read_from_head true
      tag node-problem-detector
    </source>

    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "kube-container-runtime-monitor.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-kube-container-runtime-monitor.pos
      </storage>
      read_from_head true
      tag kube-container-runtime-monitor
    </source>

    <source>
      @type systemd
      filters [{ "_SYSTEMD_UNIT": "kubelet-monitor.service" }]
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald-kubelet-monitor.pos
      </storage>
      read_from_head true
      tag kubelet-monitor
    </source>

    # Whether to include node-journal or not is determined when starting the
    # cluster. It is not changed when the cluster is already running.
    <source>
      @type systemd
      <storage>
        @type local
        path /var/run/google-fluentd/pos-files/gcp-journald.pos
      </storage>
      read_from_head true
      tag node-journal
    </source>

    <filter node-journal>
      @type grep
      <exclude>
        key _SYSTEMD_UNIT
        pattern ^(docker|containerd|kubelet|kube-node-installation|kube-node-configuration|kube-logrotate|node-problem-detector|kube-container-runtime-monitor|kubelet-monitor)\.service$
      </exclude>
    </filter>
  5.monitoring.conf: |-
    # This source is used to acquire approximate process start timestamp,process_start
    # which purpose is explained before the corresponding output plugin.
    <source>
      @type exec
      command /bin/sh -c 'date +%s'
      tag process_start
      time_format %Y-%m-%d %H:%M:%S
      keys process_start_timestamp
    </source>

    # This filter is used to convert process start timestamp to integer
    # value for correct ingestion in the prometheus output plugin.
    <filter process_start>
      @type record_transformer
      enable_ruby true
      auto_typecast true
      <record>
        process_start_timestamp ${record["process_start_timestamp"].to_i}
      </record>
    </filter>
  6.output.conf: |-
    # This match is placed before the all-matching output to provide metric
    # exporter with a process start timestamp for correct exporting of
    # cumulative metrics to Stackdriver.
    <match process_start>
      @type prometheus

      <metric>
        type gauge
        name process_start_time_seconds
        desc Timestamp of the process start in seconds
        key process_start_timestamp
      </metric>
    </match>

    # This filter allows to count the number of log entries read by fluentd
    # before they are processed by the output plugin. This in turn allows to
    # monitor the number of log entries that were read but never sent, e.g.
    # because of liveness probe removing buffer.
    <filter **>
      @type prometheus
      <metric>
        type counter
        name logging_entry_count
        desc Total number of log entries generated by either application containers or system components
      </metric>
    </filter>

    # This section is exclusive for k8s_container logs. Those come with
    # 'stderr'/'stdout' tags.
    # TODO(instrumentation): Reconsider this workaround later.
    # Trim the entries which exceed slightly less than 100KB, to avoid
    # dropping them. It is a necessity, because Stackdriver only supports
    # entries that are up to 100KB in size.
    <filter {stderr,stdout}>
      @type record_transformer
      enable_ruby true
      <record>
        message ${record['message'].length > 100000 ? "[Trimmed]#{record['message'][0..100000]}..." : record['message']}
      </record>
    </filter>

    # Do not collect fluentd's own logs to avoid infinite loops.
    <match fluent.**>
      @type null
    </match>

    # Add a unique insertId to each log entry that doesn't already have it.
    # This helps guarantee the order and prevent log duplication.
    <filter **>
      @type add_insert_ids
    </filter>

    # This filter parses the 'source' field created for glog lines into a single
    # top-level field, for proper processing by the output plugin.
    # For example, if a record includes:
    #     {"source":"handlers.go:131"},
    # then the following entry will be added to the record:
    #     {"logging.googleapis.com/sourceLocation":
    #          {"file":"handlers.go", "line":"131"}
    #     }
    <filter **>
      @type record_transformer
      enable_ruby true
      <record>
        "logging.googleapis.com/sourceLocation" ${if record.is_a?(Hash) && record.has_key?('source'); source_parts = record['source'].split(':', 2); {'file' => source_parts[0], 'line' => source_parts[1]} if source_parts.length == 2; else; nil; end}
      </record>
    </filter>


    # This section is exclusive for k8s_container logs. These logs come with
    # 'stderr'/'stdout' tags.
    # We use a separate output stanza for 'k8s_node' logs with a smaller buffer
    # because node logs are less important than user's container logs.
    <match {stderr,stdout}>
      @type google_cloud

      # Try to detect JSON formatted log entries.
      detect_json true
      # Collect metrics in Prometheus registry about plugin activity.
      enable_monitoring true
      monitoring_type prometheus
      # Allow log entries from multiple containers to be sent in the same request.
      split_logs_by_tag false
      # Set the buffer type to file to improve the reliability and reduce the memory consumption
      buffer_type file
      buffer_path /var/run/google-fluentd/buffers/kubernetes.containers.buffer
      # Set queue_full action to block because we want to pause gracefully
      # in case of the off-the-limits load instead of throwing an exception
      buffer_queue_full_action block
      # Set the chunk limit conservatively to avoid exceeding the recommended
      # chunk size of 5MB per write request.
      buffer_chunk_limit 512k
      # Cap the combined memory usage of this buffer and the one below to
      # 512KiB/chunk * (6 + 2) chunks = 4 MiB
      buffer_queue_limit 6
      # Never wait more than 5 seconds before flushing logs in the non-error case.
      flush_interval 5s
      # Never wait longer than 30 seconds between retries.
      max_retry_wait 30
      # Disable the limit on the number of retries (retry forever).
      disable_retry_limit
      # Use multiple threads for processing.
      num_threads 2
      use_grpc true
      k8s_cluster_name "#{ENV["CLUSTER_NAME"]}"
      k8s_cluster_location "#{ENV["CLUSTER_LOCATION"]}"
      # Skip timestamp adjustment as this is in a controlled environment with
      # known timestamp format. This helps with CPU usage.
      adjust_invalid_timestamps false
    </match>

    # This section is exclusive for 'gvisor' logs. These logs come with tags
    # `gvisor` tags.
    # We use a separate output stanza for 'gvisor' logs with a smaller
    # buffer because user's container application logs are more important.
    <match gvisor>
      @type google_cloud
      detect_json true
      enable_monitoring true
      monitoring_type prometheus
      # Allow log entries from multiple pods to be sent in the same request.
      split_logs_by_tag false
      buffer_type file
      buffer_path /var/run/google-fluentd/buffers/kubernetes.pod.buffer
      buffer_queue_full_action block
      buffer_chunk_limit 512k
      buffer_queue_limit 2
      flush_interval 5s
      max_retry_wait 30
      disable_retry_limit
      num_threads 2
      use_grpc true
      # Skip timestamp adjustment as this is in a controlled environment with
      # known timestamp format. This helps with CPU usage.
      adjust_invalid_timestamps false
    </match>

    # Attach local_resource_id for 'k8s_node' monitored resource.
    <filter **>
      @type record_transformer
      enable_ruby true
      <record>
        "logging.googleapis.com/local_resource_id" ${"k8s_node.#{ENV['NODE_NAME']}"}
      </record>
    </filter>

    # This section is exclusive for 'k8s_node' logs. These logs come with tags
    # that are neither 'stderr' or 'stdout'.
    # We use a separate output stanza for 'k8s_container' logs with a larger
    # buffer because user's container logs are more important than node logs.
    <match **>
      @type google_cloud

      detect_json true
      enable_monitoring true
      monitoring_type prometheus
      # Allow entries from multiple system logs to be sent in the same request.
      split_logs_by_tag false
      detect_subservice false
      buffer_type file
      buffer_path /var/run/google-fluentd/buffers/kubernetes.system.buffer
      buffer_queue_full_action block
      buffer_chunk_limit 512k
      buffer_queue_limit 2
      flush_interval 5s
      max_retry_wait 30
      disable_retry_limit
      num_threads 2
      use_grpc true
      k8s_cluster_name "#{ENV["CLUSTER_NAME"]}"
      k8s_cluster_location "#{ENV["CLUSTER_LOCATION"]}"
      # Skip timestamp adjustment as this is in a controlled environment with
      # known timestamp format. This helps with CPU usage.
      adjust_invalid_timestamps false
    </match>

  4.knative.input.conf: |-
    # This configuration file for Fluentd is used to collect the logs located
    # inside the /var/log directory of a Cloud Run on GKE / Knative container.
    # Knative mounts a emptyDir volume named 'knative-var-log' inside the
    # user container and if collection is enabled it creates a symbolic link
    # inside another emptyDir named 'knative-internal' that contains the
    # information needed for Kubernetes metadata enrichment.
    #
    # Concretely, on the host the symbolic link is:
    # /var/lib/kubelet/pods/<POD_ID>/volumes/kubernetes.io~empty-dir/knative-internal/<NAMESPACE_NAME>_<POD_NAME>_<CONTAINER_NAME>
    # ->
    # /var/lib/kubelet/pods/<POD_ID>/volumes/kubernetes.io~empty-dir/knative-var-log
    #
    # The record reformer is used to extract pod_name, namespace_name and
    # container_name from the tag and set them in a local_resource_id in the
    # format of:
    # 'k8s_container.<NAMESPACE_NAME>.<POD_NAME>.<CONTAINER_NAME>'.
    # The reformer also sets the label 'source' to the path of the log file as seen
    # from the user container point of view to be able to identify the source of
    # a log entry.
    <source>
      @type tail
      # **/*/**/* allows path expansion to go through the symbolic link and then recursively through /var/log
      path /var/lib/kubelet/pods/*/volumes/kubernetes.io~empty-dir/knative-internal/**/*/**/*
      # Save the path of the file as field 'source'
      path_key source
      pos_file /var/run/google-fluentd/pos-files/knative-var-log.pos
      # Tags at this point are in the format of:
      # knative.reform.var.lib.kubelet.pods.<POD_ID>.volumes.kubernetes.io~empty-dir.knative-internal.<NAMESPACE_NAME>_<POD_NAME>_<CONTAINER_NAME>.<PATH_INSIDE_VAR_LOG>
      tag knative.reform.*
      read_from_head true
      <parse>
        @type multi_format
        <pattern>
          format json
          time_key time
          time_format %Y-%m-%dT%H:%M:%S.%NZ
        </pattern>
        <pattern>
          format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
          time_format %Y-%m-%dT%H:%M:%S.%N%:z
        </pattern>
        <pattern>
          format none
          message_key log
        </pattern>
      </parse>
    </source>

    <filter knative.reform.**>
      @type parser
      format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<log>.*)/
      reserve_data true
      suppress_parse_error_log true
      emit_invalid_record_to_error false
      key_name log
    </filter>

    <filter knative.reform.**>
      # This plugin uses environment variables KUBERNETES_SERVICE_HOST and
      # KUBERNETES_SERVICE_PORT to talk to the API server. These environment
      # variables are added by kubelet automatically.
      @type kubernetes_metadata
      # Interval in seconds to dump cache stats locally in the Fluentd log.
      stats_interval 300
      # TTL in seconds of each cached element.
      cache_ttl 30
      # Custom regex to extract the fields
      tag_to_kubernetes_name_regexp (?<docker_id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\.volumes.kubernetes\.io~empty-dir\.knative-internal\.(?<namespace>[^_]+)_(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<container_name>user-container)\..*?$
    </filter>

    <filter knative.reform.**>
      # We have to use record_modifier because only this plugin supports complex
      # logic to modify record the way we need.
      @type record_modifier
      enable_ruby true
      <record>
        # Extract "kubernetes"->"labels" and set them as
        # "logging.googleapis.com/labels". Prefix these labels with
        # "k8s-pod" to distinguish with other labels and avoid
        # label name collision with other types of labels.
        _dummy_ ${if record.is_a?(Hash) && record.has_key?('kubernetes') && record['kubernetes'].has_key?('labels') && record['kubernetes']['labels'].is_a?(Hash); then; record['logging.googleapis.com/labels'] = record['kubernetes']['labels'].map{ |k, v| ["k8s-pod/#{k}", v]}.to_h; end; nil}
      </record>
      # Delete this dummy field and the rest of "kubernetes" and "docker".
      remove_keys _dummy_,kubernetes,docker
    </filter>

    <match knative.reform.**>
      @type record_reformer
      enable_ruby true
      <record>
        # Extract local_resource_id from tag for 'k8s_container' monitored
        # resource. The format is:
        # 'k8s_container.<namespace_name>.<pod_name>.<container_name>'.
        "logging.googleapis.com/local_resource_id" ${"k8s_container.#{tag_parts[11].gsub('_', '.')}"}
        # Rename the field 'log' to a more generic field 'message'. This way the
        # fluent-plugin-google-cloud knows to flatten the field as textPayload
        # instead of jsonPayload after extracting 'time', 'severity' and
        # 'stream' from the record.
        message ${record['log']}
        # If 'severity' is not set, assume it is INFO.
        severity ${record['severity'] || 'INFO'}
        # Set 'source' label to the path of the log file as seen from the Knative container point of view
        _dummy_ ${record["logging.googleapis.com/labels"]["source"] = "/var/log/" + record["source"].scan(/\/knative-internal\/[^\/]+\/(.*)/).last.last}
      </record>
      tag knative.stdout
      remove_keys _dummy_,log,source
    </match>

    # Detect exceptions in the log output and forward them as one log entry.
    <match knative.stdout>
      @type detect_exceptions
      remove_tag_prefix knative
      message message
      stream "logging.googleapis.com/local_resource_id"
      multiline_flush_interval 5
      max_bytes 500000
      max_lines 1000
    </match>
---
# Config map for Logging Agent output and corresponding filter plugins.
apiVersion: v1
kind: ConfigMap
metadata:
  name: logging-agent-output-config
  namespace: stackdriver-agents
data:
  google-fluentd.conf: |-
    @include config.d/*.conf

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: stackdriver-metadata-agent
    cluster-level: "true"
  name: stackdriver-metadata-agent-cluster-level
  namespace: stackdriver-agents
spec:
  replicas: 1
  selector:
    matchLabels:
      app: stackdriver-metadata-agent
      cluster-level: "true"
  template:
    metadata:
      labels:
        app: stackdriver-metadata-agent
        cluster-level: "true"
    spec:
      containers:
      - env:
        - name: CLUSTER_NAME
          valueFrom:
            configMapKeyRef:
              name: cluster-config
              key: cluster_name
        - name: CLUSTER_LOCATION
          valueFrom:
            configMapKeyRef:
              name: cluster-config
              key: cluster_location
        - name: GOOGLE_APPLICATION_CREDENTIALS
          valueFrom:
            configMapKeyRef:
              name: google-cloud-config
              key: credentials_path
        - name: PROMETHEUS_PORT
          value: "8888"
        args:
        - -logtostderr
        - -v=1
        image: gcr.io/stackdriver-agents/metadata-agent-go:1.2.1
        imagePullPolicy: IfNotPresent
        name: metadata-agent
        resources:
          requests:
            cpu: 40m
            memory: 50Mi
        ports:
        - name: metadata-agent
          containerPort: 8888
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/google-cloud/
          name: google-cloud-config
        - mountPath: /etc/ssl/certs
          name: ssl-certs
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: metadata-agent
      serviceAccountName: metadata-agent
      tolerations:
      - operator: "Exists"
        effect: "NoExecute"
      - operator: "Exists"
        effect: "NoSchedule"
      terminationGracePeriodSeconds: 5
      volumes:
      - configMap:
          defaultMode: 420
          name: google-cloud-config
        name: google-cloud-config
      - hostPath:
          path: /etc/ssl/certs
          type: Directory
        name: ssl-certs
  strategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate

---