[core.logging] Add RewriteAppender for filtering LogMeta.

docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md

@@ -8,5 +8,5 @@
 <b>Signature:</b>
 
 ```typescript
-export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RollingFileAppenderConfig;
+export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RewriteAppenderConfig | RollingFileAppenderConfig;
 ```

docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md

@@ -25,7 +25,7 @@ const page1 = await savedObjectsClient.find({
   type: 'visualization',
   sortField: 'updated_at',
   sortOrder: 'asc',
-  pit,
+  pit: { id },
 });
 const lastHit = page1.saved_objects[page1.saved_objects.length - 1];
 const page2 = await savedObjectsClient.find({

docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md

@@ -29,19 +29,16 @@ openPointInTimeForType(type: string | string[], { keepAlive, preference }?: Save
 
 
 ```ts
-const repository = coreStart.savedObjects.createInternalRepository();
-
-const { id } = await repository.openPointInTimeForType(
-  type: 'index-pattern',
-  { keepAlive: '2m' },
+const { id } = await savedObjectsClient.openPointInTimeForType(
+  type: 'visualization',
+  { keepAlive: '5m' },
 );
 const page1 = await savedObjectsClient.find({
   type: 'visualization',
   sortField: 'updated_at',
   sortOrder: 'asc',
-  pit,
+  pit: { id, keepAlive: '2m' },
 });
-
 const lastHit = page1.saved_objects[page1.saved_objects.length - 1];
 const page2 = await savedObjectsClient.find({
   type: 'visualization',
@@ -50,7 +47,6 @@ const page2 = await savedObjectsClient.find({
   pit: { id: page1.pit_id },
   searchAfter: lastHit.sort,
 });
-
 await savedObjectsClient.closePointInTime(page2.pit_id);
 
 ```

packages/kbn-logging/src/appenders.ts

@@ -15,6 +15,7 @@ import { LogRecord } from './log_record';
  */
 export interface Appender {
   append(record: LogRecord): void;
+  update(config: { appenders?: Map<string, Appender> }): void;
 }
 
 /**

src/core/server/http/integration_tests/logging.test.ts

@@ -251,7 +251,42 @@ describe('request logging', () => {
           expect(JSON.parse(meta).http.response.headers.bar).toBe('world');
         });
 
-        it('filters sensitive request headers', async () => {
+        it('filters sensitive request headers when RewriteAppender is configured', async () => {
+          root = kbnTestServer.createRoot({
+            logging: {
+              silent: true,
+              appenders: {
+                'test-console': {
+                  type: 'console',
+                  layout: {
+                    type: 'pattern',
+                    pattern: '%level|%logger|%message|%meta',
+                  },
+                },
+                rewrite: {
+                  type: 'rewrite',
+                  appenders: ['test-console'],
+                  policy: {
+                    type: 'meta',
+                    mode: 'update',
+                    properties: [
+                      { path: 'http.request.headers.authorization', value: '[REDACTED]' },
+                    ],
+                  },
+                },
+              },
+              loggers: [
+                {
+                  name: 'http.server.response',
+                  appenders: ['rewrite'],
+                  level: 'debug',
+                },
+              ],
+            },
+            plugins: {
+              initialize: false,
+            },
+          });
           const { http } = await root.setup();
 
           http.createRouter('/').post(
@@ -283,7 +318,40 @@ describe('request logging', () => {
           expect(JSON.parse(meta).http.request.headers.authorization).toBe('[REDACTED]');
         });
 
-        it('filters sensitive response headers', async () => {
+        it('filters sensitive response headers when RewriteAppender is configured', async () => {
+          root = kbnTestServer.createRoot({
+            logging: {
+              silent: true,
+              appenders: {
+                'test-console': {
+                  type: 'console',
+                  layout: {
+                    type: 'pattern',
+                    pattern: '%level|%logger|%message|%meta',
+                  },
+                },
+                rewrite: {
+                  type: 'rewrite',
+                  appenders: ['test-console'],
+                  policy: {
+                    type: 'meta',
+                    mode: 'update',
+                    properties: [{ path: 'http.response.headers.set-cookie', value: '[REDACTED]' }],
+                  },
+                },
+              },
+              loggers: [
+                {
+                  name: 'http.server.response',
+                  appenders: ['rewrite'],
+                  level: 'debug',
+                },
+              ],
+            },
+            plugins: {
+              initialize: false,
+            },
+          });
           const { http } = await root.setup();
 
           http.createRouter('/').post(

src/core/server/http/logging/get_response_log.test.ts

@@ -148,54 +148,6 @@ describe('getEcsResponseLog', () => {
     expect(result.http.response.status_code).toBe(400);
   });
 
-  describe('filters sensitive headers', () => {
-    test('redacts Authorization and Cookie headers by default', () => {
-      const req = createMockHapiRequest({
-        headers: { authorization: 'a', cookie: 'b', 'user-agent': 'hi' },
-        response: { headers: { 'content-length': 123, 'set-cookie': 'c' } },
-      });
-      const result = getEcsResponseLog(req, logger);
-      expect(result.http.request.headers).toMatchInlineSnapshot(`
-        Object {
-          "authorization": "[REDACTED]",
-          "cookie": "[REDACTED]",
-          "user-agent": "hi",
-        }
-      `);
-      expect(result.http.response.headers).toMatchInlineSnapshot(`
-        Object {
-          "content-length": 123,
-          "set-cookie": "[REDACTED]",
-        }
-      `);
-    });
-
-    test('does not mutate original headers', () => {
-      const reqHeaders = { authorization: 'a', cookie: 'b', 'user-agent': 'hi' };
-      const resHeaders = { headers: { 'content-length': 123, 'set-cookie': 'c' } };
-      const req = createMockHapiRequest({
-        headers: reqHeaders,
-        response: { headers: resHeaders },
-      });
-      getEcsResponseLog(req, logger);
-      expect(reqHeaders).toMatchInlineSnapshot(`
-        Object {
-          "authorization": "a",
-          "cookie": "b",
-          "user-agent": "hi",
-        }
-      `);
-      expect(resHeaders).toMatchInlineSnapshot(`
-        Object {
-          "headers": Object {
-            "content-length": 123,
-            "set-cookie": "c",
-          },
-        }
-      `);
-    });
-  });
-
   describe('ecs', () => {
     test('specifies correct ECS version', () => {
       const req = createMockHapiRequest();

src/core/server/http/logging/get_response_log.ts

@@ -15,21 +15,6 @@ import { EcsEvent, Logger } from '../../logging';
 import { getResponsePayloadBytes } from './get_payload_size';
 
 const ECS_VERSION = '1.7.0';
-const FORBIDDEN_HEADERS = ['authorization', 'cookie', 'set-cookie'];
-const REDACTED_HEADER_TEXT = '[REDACTED]';
-
-// We are excluding sensitive headers by default, until we have a log filtering mechanism.
-function redactSensitiveHeaders(
-  headers?: Record<string, string | string[]>
-): Record<string, string | string[]> {
-  const result = {} as Record<string, string | string[]>;
-  if (headers) {
-    for (const key of Object.keys(headers)) {
-      result[key] = FORBIDDEN_HEADERS.includes(key) ? REDACTED_HEADER_TEXT : headers[key];
-    }
-  }
-  return result;
-}
 
 /**
  * Converts a hapi `Request` into ECS-compliant `LogMeta` for logging.
@@ -66,15 +51,15 @@ export function getEcsResponseLog(request: Request, log: Logger): LogMeta {
         mime_type: request.mime,
         referrer: request.info.referrer,
         // @ts-expect-error Headers are not yet part of ECS: https://github.com/elastic/ecs/issues/232.
-        headers: redactSensitiveHeaders(request.headers),
+        headers: request.headers,
       },
       response: {
         body: {
           bytes,
         },
         status_code,
         // @ts-expect-error Headers are not yet part of ECS: https://github.com/elastic/ecs/issues/232.
-        headers: redactSensitiveHeaders(responseHeaders),
+        headers: responseHeaders,
         // responseTime is a custom non-ECS field
         responseTime: !isNaN(responseTime) ? responseTime : undefined,
       },

src/core/server/legacy/logging/appenders/legacy_appender.ts

@@ -47,6 +47,13 @@ export class LegacyAppender implements DisposableAppender {
     this.loggingServer.log(record);
   }
 
+  /**
+   * Updates `LegacyAppender` configuration.
+   */
+  public update() {
+    // noop
+  }
+
   public dispose() {
     this.loggingServer.stop();
   }

src/core/server/logging/appenders/appenders.ts

@@ -17,6 +17,7 @@ import {
 import { Layouts } from '../layouts/layouts';
 import { ConsoleAppender, ConsoleAppenderConfig } from './console/console_appender';
 import { FileAppender, FileAppenderConfig } from './file/file_appender';
+import { RewriteAppender, RewriteAppenderConfig } from './rewrite/rewrite_appender';
 import {
   RollingFileAppender,
   RollingFileAppenderConfig,
@@ -32,6 +33,7 @@ export const appendersSchema = schema.oneOf([
   ConsoleAppender.configSchema,
   FileAppender.configSchema,
   LegacyAppender.configSchema,
+  RewriteAppender.configSchema,
   RollingFileAppender.configSchema,
 ]);
 
@@ -40,6 +42,7 @@ export type AppenderConfigType =
   | ConsoleAppenderConfig
   | FileAppenderConfig
   | LegacyAppenderConfig
+  | RewriteAppenderConfig
   | RollingFileAppenderConfig;
 
 /** @internal */
@@ -57,6 +60,8 @@ export class Appenders {
         return new ConsoleAppender(Layouts.create(config.layout));
       case 'file':
         return new FileAppender(Layouts.create(config.layout), config.fileName);
+      case 'rewrite':
+        return new RewriteAppender(config);
       case 'rolling-file':
         return new RollingFileAppender(config);
       case 'legacy-appender':

src/core/server/logging/appenders/buffer/buffer_appender.ts

@@ -27,6 +27,13 @@ export class BufferAppender implements DisposableAppender {
     this.buffer.push(record);
   }
 
+  /**
+   * Updates `BufferAppender` configuration.
+   */
+  public update() {
+    // noop
+  }
+
   /**
    * Clears buffer and returns all records that it had.
    */

src/core/server/logging/appenders/console/console_appender.ts

@@ -43,6 +43,13 @@ export class ConsoleAppender implements DisposableAppender {
     console.log(this.layout.format(record));
   }
 
+  /**
+   * Updates `ConsoleAppender` configuration.
+   */
+  public update() {
+    // noop
+  }
+
   /**
    * Disposes `ConsoleAppender`.
    */

src/core/server/logging/appenders/file/file_appender.ts

@@ -56,6 +56,13 @@ export class FileAppender implements DisposableAppender {
     this.outputStream.write(`${this.layout.format(record)}\n`);
   }
 
+  /**
+   * Updates `FileAppender` configuration.
+   */
+  public update() {
+    // noop
+  }
+
   /**
    * Disposes `FileAppender`. Waits for the underlying file stream to be completely flushed and closed.
    */

src/core/server/logging/appenders/rewrite/mocks.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { RewritePolicy } from './policies/policy';
+
+const createPolicyMock = () => {
+  const mock: jest.Mocked<RewritePolicy> = {
+    transform: jest.fn((x) => x),
+  };
+  return mock;
+};
+
+export const rewriteAppenderMocks = {
+  createPolicy: createPolicyMock,
+};

src/core/server/logging/appenders/rewrite/policies/index.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { schema } from '@kbn/config-schema';
+import { assertNever } from '@kbn/std';
+import { RewritePolicy } from './policy';
+import { MetaRewritePolicy, MetaRewritePolicyConfig, metaRewritePolicyConfigSchema } from './meta';
+
+export { RewritePolicy };
+
+/**
+ * Available rewrite policies which specify what part of a {@link LogRecord}
+ * can be modified.
+ */
+export type RewritePolicyConfig = MetaRewritePolicyConfig;
+
+export const rewritePolicyConfigSchema = schema.oneOf([metaRewritePolicyConfigSchema]);
+
+export const createRewritePolicy = (config: RewritePolicyConfig): RewritePolicy => {
+  switch (config.type) {
+    case 'meta':
+      return new MetaRewritePolicy(config);
+    default:
+      return assertNever(config.type);
+  }
+};

src/core/server/logging/appenders/rewrite/policies/meta/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export {
+  MetaRewritePolicy,
+  MetaRewritePolicyConfig,
+  metaRewritePolicyConfigSchema,
+} from './meta_policy';