From 9e25091abb667f269fdb30a2e84976e71278e25d Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Fri, 15 Nov 2024 16:58:14 +0000 Subject: [PATCH 1/8] FP-3066 + FP-3034 --- CHANGELOG.md | 5 + package.json | 2 +- src/api/Features/index.js | 4 +- src/api/Logs/index.js | 249 +++++++++++++++++++++++++++ src/api/RobotManager/Robot.ts | 127 +------------- src/api/RobotManager/RobotManager.ts | 69 -------- src/api/RobotManager/Utils/Utils.ts | 150 ---------------- src/api/Utils/constants.js | 23 ++- src/api/index.ts | 2 + src/models/robot.ts | 11 -- 10 files changed, 276 insertions(+), 366 deletions(-) create mode 100644 src/api/Logs/index.js delete mode 100644 src/api/RobotManager/Utils/Utils.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 79dcb6a9..8d17147d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# TBD + +- [FP-3066](https://movai.atlassian.net/browse/FP-3066): Fleetboard: Logs of the workers are stored in the manager but the fleetboard is not able to display them correctly +- [FP-3034](https://movai.atlassian.net/browse/FP-3034): Logs - Do not select backend by default + # 1.2.4 - [FP-3073](https://movai.atlassian.net/browse/FP-3073): Frontend says robot offline. Only fixable on browser refresh diff --git a/package.json b/package.json index 40b7e366..90a3f25a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@mov-ai/mov-fe-lib-core", - "version": "1.2.4-0", + "version": "1.3.3-0", "description": "The Mov.AI's core frontend library.", "keywords": [ "frontend", diff --git a/src/api/Features/index.js b/src/api/Features/index.js index f8535a59..b289b737 100644 --- a/src/api/Features/index.js +++ b/src/api/Features/index.js @@ -27,7 +27,7 @@ if (!window.mock) { const Features = { get: (key) => features[key], - set: (key) => { + set: (key, value) => { features[key] = value; }, enable: (key) => { @@ -38,4 +38,6 @@ const Features = { }, }; +globalThis.Features = Features; + export default Features; diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js new file mode 100644 index 00000000..eed62f98 --- /dev/null +++ b/src/api/Logs/index.js @@ -0,0 +1,249 @@ +import { webSocketOpen } from "../WebSocket"; +import Features from "./../Features"; +import Rest from "../Rest/Rest"; +import { + DEFAULT_LEVELS, + DEFAULT_SERVICE, + MAX_LOG_LIMIT, +} from "./../Utils/constants"; + +const MAX_FETCH_LOGS = 200000; +let logsDataGlobal = []; + +/** + * Tranform log from the format received from the API to the format + * required to be rendered + * @returns {array} Transformed log + */ +function transformLog(log, _index, _data, ts_multiplier = 1000) { + const timestamp = ts_multiplier * log.time; + const date = new Date(timestamp); + return { + ...log, + timestamp, + time: date.toLocaleTimeString(), + date: date.toLocaleDateString(), + key: log.message + timestamp, + }; +} + +/** + * Remove duplicates from logs for the second overlaping the + * current and the last request + * @returns {array} Concatenated logs without duplicates + */ +export function logsDedupe(oldLogs, data) { + if (!data.length) return oldLogs; + + // date of the oldest log received in the current request + const oldDate = data[data.length - 1].timestamp; + // map to store the old logs of the overlaped second + let map = {}; + + // iter over old logs with last timestamp of the new logs + // and put in a map + for (let i = 0; i < oldLogs.length && oldLogs[i].timestamp === oldDate; i++) + map[oldLogs[i].message] = oldLogs[i]; + + // array to store logs from overlap second which had not + // been sent before + let newSecOverlap = []; + let z; + + // iter over new logs (oldest to latest) with last timestamp, + // check if present in last map + // - if not, push + for (z = data.length - 1; z >= 0 && data[z].timestamp === oldDate; z--) + if (!map[data[z].message]) newSecOverlap.push(data[z]); + + // cut new logs up to z, concat with the deduped ones + // and the old logs up to i + return data.slice(0, z + 1).concat(newSecOverlap.reverse(), oldLogs); +} + +function getRequestList(label, values) { + return values?.length + ? label + "=" + values.map((el) => el.label).join() + : ""; +} + +function getRequestString(label, value) { + return value ? label + "=" + value : ""; +} + +function getRequestDate(label, value) { + return getRequestString(label, (value ?? 0) / 1000); +} + +/** + * Get Logs parameter string + * @param {LogQueryParam} queryParam : Object to construct query string + * @returns {string} query parameter string + */ +function getLogsParam(queryParam = {}) { + return [ + getRequestString("limit", queryParam?.limit || MAX_LOG_LIMIT), + getRequestList("level", queryParam.level?.selected), + getRequestList("services", queryParam.service?.selected), + getRequestList("tags", queryParam.tag?.selected), + getRequestString("message", queryParam.searchMessage), + getRequestDate("fromDate", queryParam.date?.from), + getRequestDate("toDate", queryParam.date?.to), + getRequestList("robots", queryParam.robot?.selected), + ] + .filter((a) => !!a) + .join("&"); +} + +async function getLogs(maximum = 0) { + const path = + "v1/logs/?" + + getLogsParam({ + limit: MAX_FETCH_LOGS, + date: { + from: logsDataGlobal.length ? logsDataGlobal[0].timestamp : null, + to: null, + }, + }); + + const response = await Rest.get({ path }); + + const data = response?.data || []; + const oldLogs = logsDataGlobal || []; + const newLogs = data.map(transformLog); + + return (logsDataGlobal = ( + Features.get("noLogsDedupe") + ? newLogs.concat(oldLogs) + : logsDedupe(oldLogs, newLogs) + ).slice(-maximum)); +} + +function noSelection(obj) { + for (let key in obj) if (obj[key]) return false; + return true; +} + +function matchTags(tags, item) { + for (const tag in tags) + if (item[tag] !== undefined) continue; + else return false; + return true; +} + +let singleton = null; + +function websocketTransform(original) { + const { message } = original; + + let tags = {}; + let realMessage = message; + + if (message.charAt(0) === "[") { + const braketParts = message.substring(1).split("] "); + + tags = braketParts[0].split("|").reduce((a, item) => { + const [key, value] = item.split(":"); + return { + ...a, + [key]: value, + }; + }, {}); + + realMessage = braketParts[1]; + } + + return { + ...original, + message: realMessage, + ...tags, + }; +} + +export default class Logs { + static CONSTANTS = { + DEFAULT_LEVELS, + DEFAULT_SERVICE, + }; + + // change maximum to another number to limit the amount of kept logs + constructor(maximum = 0) { + if (singleton) return singleton; + + singleton = this; + this.subs = new Map(); + this.refresh(); + this.maximum = maximum; + } + + clear() { + logsDataGlobal = []; + this.update(); + } + + refresh() { + this.streaming = Features.get("logStreaming"); + this.clear(); + + if (this.streaming) { + const params = new URLSearchParams(); + const sock = webSocketOpen({ path: "/ws/logs", params }); + sock.onmessage = (msg) => { + const item = websocketTransform(JSON.parse(msg?.data ?? {})); + logsDataGlobal = [transformLog(item, 0, [item], 0.000001)] + .concat(logsDataGlobal) + .slice(-maximum); + this.update(); + }; + getLogs(this.maximum).then(() => this.update()); + } else this.getLogs(); + } + + async getLogs() { + try { + await getLogs(this.maximum); + this.update(); + } catch (e) { + console.log("Failed getting logs", e); + } + setTimeout(() => this.getLogs(), 3000); + } + + get() { + return logsDataGlobal; + } + + filter(query = {}) { + const { + levels = DEFAULT_LEVELS, + service = DEFAULT_SERVICE, + tags = {}, + robots = {}, + selectedFromDate = null, + selectedToDate = null, + message = "", + } = query; + + return logsDataGlobal.filter( + (item) => + (levels[item.level] || noSelection(levels)) && + (service[item.service] || noSelection(service)) && + (matchTags(tags, item) || noSelection(tags)) && + (item.message || "").includes(message) && + (robots[item.robot] || noSelection(robots)) && + (!selectedFromDate || item.timestamp >= selectedFromDate) && + (!selectedToDate || item.timestamp <= selectedToDate), + ); + } + + subscribe(callback) { + this.subs.set(callback, true); + return () => this.subs.delete(callback); + } + + update() { + for (const [sub] of this.subs) sub(logsDataGlobal); + } +} + +globalThis.Logs = Logs; diff --git a/src/api/RobotManager/Robot.ts b/src/api/RobotManager/Robot.ts index 49e23ae8..5c48ca0d 100644 --- a/src/api/RobotManager/Robot.ts +++ b/src/api/RobotManager/Robot.ts @@ -1,10 +1,6 @@ import _cloneDeep from "lodash/cloneDeep"; import { LoadRobotParam, - Log, - Tasks, - LogData, - Logger, RobotMap, RobotModel, UpdateRobotParam, @@ -14,12 +10,7 @@ import { Alert, Alerts, } from "../../models"; -import { - LOGGER_STATUS, - EMPTY_FUNCTION, - DEFAULT_ROBOT_TASKS, - TIME_TO_OFFLINE, -} from "../Utils/constants"; +import { EMPTY_FUNCTION, TIME_TO_OFFLINE } from "../Utils/constants"; import DocumentV2 from "../Document/DocumentV2"; import MasterDB from "../Database/MasterDB"; import * as Utils from "./../Utils/Utils"; @@ -33,9 +24,6 @@ class Robot { private name?: RobotModel["RobotName"]; private previousData: RobotModel; private lastUpdate: Date; - private logs: Array; - private logger: Logger; - private logSubscriptions: SubscriptionManager; private dataSubscriptions: SubscriptionManager; private onGetIPCallback: Function; private api: DocumentV2; @@ -52,12 +40,6 @@ class Robot { this.data = { ...data, Online: true }; this.previousData = this.data; this.lastUpdate = new Date(); - this.logs = []; - this.logger = { - status: LOGGER_STATUS.init, - time: 3000, - }; - this.logSubscriptions = {}; this.dataSubscriptions = {}; this.onGetIPCallback = EMPTY_FUNCTION; this.api = Document.factory( @@ -180,22 +162,6 @@ class Robot { this.subscribe({ property: "IP", onLoad: this._loadIP(this) }); } - /** - * Start robot logger - */ - startLogger() { - this.logger.status = LOGGER_STATUS.running; - this._getLogs(); - } - - /** - * Stop robot logger - */ - stopLogger() { - this.logger.status = LOGGER_STATUS.paused; - clearTimeout(this.logger.timeout as NodeJS.Timeout); - } - /** * Subscribe to changes in robot's data * @param {Function} callback: Function to be called on get logs @@ -255,28 +221,6 @@ class Robot { return keys; } - /** - * Subscribe to the robot logs - * @param {Function} callback: Function to be called on get logs - */ - subscribeToLogs(callback: Function) { - const subscriptionId = Utils.randomGuid(); - this.logSubscriptions[subscriptionId] = { send: callback }; - if (this.logger.status !== LOGGER_STATUS.running) this.startLogger(); - else if (this.logs) callback(this.logs); - return subscriptionId; - } - - /** - * Unsubscribe to the robot logs - * @param {string} subscriptionId: Subscription id that needs to be canceled - */ - unsubscribeToLogs(subscriptionId: string) { - if (!subscriptionId || !this.logSubscriptions[subscriptionId]) return; - delete this.logSubscriptions[subscriptionId]; - if (Object.keys(this.logSubscriptions).length === 0) this.stopLogger(); - } - /** * Unsubscribe to the robot data * @@ -287,81 +231,12 @@ class Robot { delete this.dataSubscriptions[subscriptionId]; } - /** - * Refresh logs - */ - refreshLogs() { - clearTimeout(this.logger.timeout as NodeJS.Timeout); - this._getLogs(); - } - - /** - * Get robot current and previous tasks - * @returns {Promise} Returns previous/current robot tasks - */ - async getTasks(): Promise { - const path = `v1/logs/?limit=2&level=info&tags=ui&robots=${this.name}`; - return Rest.get({ path }) - .then((res: Log) => { - if (!res?.data?.length) { - return DEFAULT_ROBOT_TASKS; - } - // Return tasks - return { - currentTask: res.data[0]?.message || DEFAULT_ROBOT_TASKS.currentTask, - previousTask: - res.data[1]?.message || DEFAULT_ROBOT_TASKS.previousTask, - }; - }) - .catch((error: Error) => { - console.warn("Failed to get tasks", error); - return DEFAULT_ROBOT_TASKS; - }); - } - //======================================================================================== /* * * Private functions * * */ //======================================================================================== - /** - * Get robot logs - */ - private _getLogs() { - if (Object.keys(this.logSubscriptions).length === 0) return; // Stop if there's no active subscriptions - if (this.logger.status !== LOGGER_STATUS.running) return; // Or if logger status is not "running" - if (!this.name) return; // Or if robot has no name - - // Get logs from server - const path = `v1/logs/?limit=20&level=info,error,warning,critical&tags=ui&robots=${this.name}`; - Rest.get({ path }) - .then((response: Log) => { - if (!response || !response.data) return; - // Cache log data and send response to active subscriptions - this.logs = response.data; - for (const key in this.logSubscriptions) { - this.logSubscriptions[key].send(response.data); - } - // Enqueue next request - this._enqueueNextRequest(); - }) - .catch((err: Error) => { - // Enqueue next request - this.logger.time += 1000; - this._enqueueNextRequest(); - console.warn("Failed log request", err); - }); - } - - /** - * Enqueue next request to get logs - */ - private _enqueueNextRequest() { - clearTimeout(this.logger.timeout as NodeJS.Timeout); - this.logger.timeout = setTimeout(() => this._getLogs(), this.logger.time); - } - /** * Function to be called when robot IP subscribed loads * diff --git a/src/api/RobotManager/RobotManager.ts b/src/api/RobotManager/RobotManager.ts index d93e2466..6d79f345 100644 --- a/src/api/RobotManager/RobotManager.ts +++ b/src/api/RobotManager/RobotManager.ts @@ -6,23 +6,11 @@ import { EMPTY_FUNCTION, HEARTBEAT_TIMEOUT, SET_WS_EVENTS, - MAX_LOG_LIMIT, } from "../Utils/constants"; import Robot from "./Robot"; -import Rest from "../Rest/Rest"; -import { webSocketOpen } from "../WebSocket"; -import { - getRequestDate, - getRequestLevels, - getRequestMessage, - getRequestRobots, - getRequestService, - getRequestTags, -} from "./Utils/Utils"; import { CachedRobots, LoadRobotParam, - LogQueryParam, RobotModel, SubscriptionManager, UpdateRobotParam, @@ -320,63 +308,6 @@ class RobotManager { robot.sendUpdates(event); }); }; - - //======================================================================================== - /* * - * Static Methods * - * */ - //======================================================================================== - /** - * Get Logs params - * @param {LogQueryParam} queryParam : Object to construct query string - * @returns {string} query parameter string - */ - static getLogsParam(queryParam: LogQueryParam): string { - // Get request parameters - const _limit = queryParam?.limit || MAX_LOG_LIMIT; - const _levels = getRequestLevels( - queryParam?.level?.selected || [], - queryParam?.level?.list, - ); - const _services = getRequestService(queryParam?.service?.selected); - const _tags = getRequestTags(queryParam?.tag?.selected); - const _message = getRequestMessage(queryParam?.searchMessage); - const _dates = getRequestDate(queryParam?.date?.from, queryParam?.date?.to); - const _robots = getRequestRobots(queryParam?.robot?.selected); - return [_limit, _levels, _services, _dates, _tags, _message, _robots].join( - "", - ); - } - - /** - * Open Websocket connection to get the logs - * @param {LogQueryParam} queryParam : Object to construct query string - * @returns {Promise} Request promise - */ - static openLogs(queryParam: LogQueryParam): WebSocket { - const splits = RobotManager.getLogsParam(queryParam).split("&"); - let params = new URLSearchParams(); - for (const split in splits) { - const [key, value] = split.split("="); - params.set(value ? key : "limit", value ?? key); - } - return webSocketOpen({ path: "/ws/logs", params }); - } - - //======================================================================================== - /* * - * Static Methods * - * */ - //======================================================================================== - /** - * Get Logs for multiple robots - * @param {LogQueryParam} queryParam : Object to construct query string - * @returns {Promise} Request promise - */ - static async getLogs(queryParam: LogQueryParam): Promise { - const path = "v1/logs/?limit=" + RobotManager.getLogsParam(queryParam); - return Rest.get({ path }); - } } export default RobotManager; diff --git a/src/api/RobotManager/Utils/Utils.ts b/src/api/RobotManager/Utils/Utils.ts deleted file mode 100644 index 709cf9b5..00000000 --- a/src/api/RobotManager/Utils/Utils.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { LogLevel, LogTag, TimestampQuery } from "../../../models"; -import { SERVICE_LIST } from "../../Utils/constants"; - -//======================================================================================== -/* * - * Private Functions * - * */ -//======================================================================================== - -/** - * @private Get date disconsidering seconds - * @param date : Date to parse - * @returns Timestamp of date without seconds - */ -function getDateWithoutSeconds(date: number): number { - return (date / 1000) | 0; -} - -//======================================================================================== -/* * - * Public Functions * - * */ -//======================================================================================== - -/** - * Converts the levels in the state for the string for the request - * Input: ["INFO", "DEBUG"] - * Output: "level=info,debug" - * @param {Array} selectedLevels : Selected levels from filter - * @param {Array} levelsList : All levels list available - * @returns {string} Request query param string for levels - */ -export function getRequestLevels( - selectedLevels: Array, - levelsList: Array = [], -): string { - if ( - Array.isArray(selectedLevels) && - (selectedLevels.length === levelsList.length || selectedLevels.length === 0) - ) { - return ""; - } - - try { - const level = selectedLevels.map((el) => el.toLowerCase()).join(); - return `&level=${level}`; - } catch (error) { - console.warn("Error Requesting Level", error); - return ""; - } -} - -/** - * Converts array of selected services into request param - * @param selectedService : Array of selected services - * @returns {string} Request query param string for services - */ -export function getRequestService(selectedService: Array = []): string { - if ( - (Array.isArray(selectedService) && - selectedService.length === SERVICE_LIST.length) || - selectedService.length === 0 - ) { - return ""; - } - - try { - const services = selectedService.map((el) => el.toLowerCase()).join(); - return `&services=${services}`; - } catch (error) { - console.warn("Error Requesting Service", error); - return ""; - } -} - -/** - * Convert date timestamp into request query params - * @param selectedFromDate : Logs "From" date - * @param selectedToDate : Logs "To" date - * @returns {string} Request query param string for dates - */ -export function getRequestDate( - selectedFromDate: TimestampQuery | undefined, - selectedToDate: TimestampQuery | undefined, -): string { - let res = ""; - try { - if (selectedFromDate) { - res += `&fromDate=${getDateWithoutSeconds(selectedFromDate)}`; - } - if (selectedToDate) { - res += `&toDate=${getDateWithoutSeconds(selectedToDate) + 59}`; - } - return res; - } catch (error) { - console.warn("Error Requesting Data", error); - return ""; - } -} - -/** - * Converts the levels in the state for the string for the request - * Input: [{ key: 0, label: "ui" }, { key: 1, label: "tasks" }] - * Output: "&tags=ui,tasks" - * @param selectedTags : Array with selected tags - * @returns {string} Request query param string for tags - */ -export function getRequestTags(selectedTags: Array = []): string { - // Return empty string if no tag is added - if (!selectedTags.length) { - return ""; - } - // Format tags to URL parameter - const tags = selectedTags.map((el) => el.label).join(); - return `&tags=${tags}`; -} - -/** - * Convert array of selected robots into query param string - * @param {Array} selectedRobots : Selected robot names - * @returns {string} Request query param string for robots - */ -export function getRequestRobots(selectedRobots: Array = []): string { - // Return empty string if no tag is added - if (!selectedRobots.length) { - return ""; - } - // Format tags to URL parameter - const robots = selectedRobots.join(); - return `&robots=${robots}`; -} - -// Converts the levels in the state for the string for the request -// Input: "r'started\.$" -// Output: "&message=r'started\.$" -/** - * Converts the levels in the state for the string for the request - * Input: "r'started\.$" - * Output: "&message=r'started\.$" - * @param {string} message : Search message - * @returns {string} Request query param string for message - */ -export function getRequestMessage(message: string = ""): string { - if (!message || message === "") { - return ""; - } - // Parse message to URL and return - const parsedMessageToUrl = message.replace(/ /g, "+"); - return `&message=${parsedMessageToUrl}`; -} diff --git a/src/api/Utils/constants.js b/src/api/Utils/constants.js index 176567fb..f1b724bc 100644 --- a/src/api/Utils/constants.js +++ b/src/api/Utils/constants.js @@ -9,14 +9,6 @@ export const BROADCAST_EVENTS = { RPL_OPENDOC: "RPL-OPENDOC", }; -// Robot Logger enum -export const LOGGER_STATUS = { - init: 0, - running: 1, - paused: 2, - terminated: 3, -}; - export const ALPHANUMERIC_REGEX = /^[\w][0-9A-Za-z-]*(_[0-9A-Za-z-]+)*[_]?$/; export const REQUEST_STATUS = { @@ -48,6 +40,21 @@ export const SERVICE_LABEL = { haproxy: "ha-proxy", }; +export const DEFAULT_SERVICE = Object.keys(SERVICE_LABEL).reduce( + (a, item) => ({ [item]: false, ...a }), + { + spawner: true, + }, +); + +export const DEFAULT_LEVELS = { + INFO: true, + WARNING: false, + DEBUG: false, + ERROR: true, + CRITICAL: true, +}; + export const VAR_SCOPES = { GLOBAL: "global", FLEET: "fleet", diff --git a/src/api/index.ts b/src/api/index.ts index bad74df8..ffd172f8 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -25,6 +25,7 @@ import Snapshot from "./Snapshot/Snapshot"; import Rest from "./Rest/Rest"; import { ROSBridge, MainROSBridge } from "./ROSBridge/ROSBridge"; import Role from "./Role/Role"; +import Logs from "./Logs"; export { Acl, @@ -54,4 +55,5 @@ export { Rest, ROSBridge, MainROSBridge, + Logs, }; diff --git a/src/models/robot.ts b/src/models/robot.ts index 55a36e87..b4dea9ee 100644 --- a/src/models/robot.ts +++ b/src/models/robot.ts @@ -51,17 +51,6 @@ export interface LoadRobotParam extends SubscriberLoadParam { value: RobotMap; } -//======================================================================================== -/* * - * Tasks * - * */ -//======================================================================================== - -export interface Tasks { - currentTask: string; - previousTask: string; -} - //======================================================================================== /* * * Logs * From 94f9b3b7b1e2965d9ea6e915434f2752ca9ef83a Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Fri, 6 Dec 2024 12:31:18 +0000 Subject: [PATCH 2/8] Interval trees are not so nice - and they solve problems --- package.json | 1 + pnpm-lock.yaml | 36 ++++---- src/api/Logs/index.js | 189 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 186 insertions(+), 40 deletions(-) diff --git a/package.json b/package.json index 90a3f25a..b1804d37 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ }, "dependencies": { "@babel/runtime": "^7.15.4", + "@flatten-js/interval-tree": "^1.1.3", "fast-equals": "^5.0.1", "js-yaml": "^4.1.0", "jwt-decode": "^3.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81fe447c..8763ce22 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@babel/runtime': specifier: ^7.15.4 version: 7.23.9 + '@flatten-js/interval-tree': + specifier: ^1.1.3 + version: 1.1.3 fast-equals: specifier: ^5.0.1 version: 5.0.1 @@ -74,7 +77,7 @@ importers: version: 28.1.3(@babel/core@7.23.9) babel-loader: specifier: ^8.2.2 - version: 8.3.0(@babel/core@7.23.9)(webpack@5.90.1(webpack-cli@4.10.0)) + version: 8.3.0(@babel/core@7.23.9)(webpack@5.90.1) express: specifier: ^4.18.1 version: 4.18.2 @@ -104,13 +107,13 @@ importers: version: 28.0.8(@babel/core@7.23.9)(@jest/types@28.1.3)(babel-jest@28.1.3(@babel/core@7.23.9))(jest@28.1.3(@types/node@17.0.45))(typescript@4.9.5) ts-loader: specifier: ^9.3.0 - version: 9.5.1(typescript@4.9.5)(webpack@5.90.1(webpack-cli@4.10.0)) + version: 9.5.1(typescript@4.9.5)(webpack@5.90.1) typescript: specifier: ^4.7.2 version: 4.9.5 url-loader: specifier: ^4.1.1 - version: 4.1.1(webpack@5.90.1(webpack-cli@4.10.0)) + version: 4.1.1(webpack@5.90.1) webpack: specifier: ^5.53.0 version: 5.90.1(webpack-cli@4.10.0) @@ -771,6 +774,9 @@ packages: resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==} engines: {node: '>=10.0.0'} + '@flatten-js/interval-tree@1.1.3': + resolution: {integrity: sha512-xhFWUBoHJFF77cJO1D6REjdgJEMRf2Y2Z+eKEPav8evGKcLSnj1ud5pLXQSbGuxF3VSvT1rWhMfVpXEKJLTL+A==} + '@istanbuljs/load-nyc-config@1.1.0': resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} engines: {node: '>=8'} @@ -3863,6 +3869,8 @@ snapshots: '@discoveryjs/json-ext@0.5.7': {} + '@flatten-js/interval-tree@1.1.3': {} + '@istanbuljs/load-nyc-config@1.1.0': dependencies: camelcase: 5.3.1 @@ -4288,17 +4296,17 @@ snapshots: '@webassemblyjs/ast': 1.11.6 '@xtuc/long': 4.2.2 - '@webpack-cli/configtest@1.2.0(webpack-cli@4.10.0(webpack@5.90.1))(webpack@5.90.1(webpack-cli@4.10.0))': + '@webpack-cli/configtest@1.2.0(webpack-cli@4.10.0)(webpack@5.90.1)': dependencies: webpack: 5.90.1(webpack-cli@4.10.0) webpack-cli: 4.10.0(webpack@5.90.1) - '@webpack-cli/info@1.5.0(webpack-cli@4.10.0(webpack@5.90.1))': + '@webpack-cli/info@1.5.0(webpack-cli@4.10.0)': dependencies: envinfo: 7.11.0 webpack-cli: 4.10.0(webpack@5.90.1) - '@webpack-cli/serve@1.7.0(webpack-cli@4.10.0(webpack@5.90.1))': + '@webpack-cli/serve@1.7.0(webpack-cli@4.10.0)': dependencies: webpack-cli: 4.10.0(webpack@5.90.1) @@ -4397,7 +4405,7 @@ snapshots: transitivePeerDependencies: - supports-color - babel-loader@8.3.0(@babel/core@7.23.9)(webpack@5.90.1(webpack-cli@4.10.0)): + babel-loader@8.3.0(@babel/core@7.23.9)(webpack@5.90.1): dependencies: '@babel/core': 7.23.9 find-cache-dir: 3.3.2 @@ -6198,7 +6206,7 @@ snapshots: ansi-escapes: 4.3.2 supports-hyperlinks: 2.3.0 - terser-webpack-plugin@5.3.10(webpack@5.90.1(webpack-cli@4.10.0)): + terser-webpack-plugin@5.3.10(webpack@5.90.1): dependencies: '@jridgewell/trace-mapping': 0.3.22 jest-worker: 27.5.1 @@ -6258,7 +6266,7 @@ snapshots: '@jest/types': 28.1.3 babel-jest: 28.1.3(@babel/core@7.23.9) - ts-loader@9.5.1(typescript@4.9.5)(webpack@5.90.1(webpack-cli@4.10.0)): + ts-loader@9.5.1(typescript@4.9.5)(webpack@5.90.1): dependencies: chalk: 4.1.2 enhanced-resolve: 5.15.0 @@ -6306,7 +6314,7 @@ snapshots: dependencies: punycode: 2.3.1 - url-loader@4.1.1(webpack@5.90.1(webpack-cli@4.10.0)): + url-loader@4.1.1(webpack@5.90.1): dependencies: loader-utils: 2.0.4 mime-types: 2.1.35 @@ -6346,9 +6354,9 @@ snapshots: webpack-cli@4.10.0(webpack@5.90.1): dependencies: '@discoveryjs/json-ext': 0.5.7 - '@webpack-cli/configtest': 1.2.0(webpack-cli@4.10.0(webpack@5.90.1))(webpack@5.90.1(webpack-cli@4.10.0)) - '@webpack-cli/info': 1.5.0(webpack-cli@4.10.0(webpack@5.90.1)) - '@webpack-cli/serve': 1.7.0(webpack-cli@4.10.0(webpack@5.90.1)) + '@webpack-cli/configtest': 1.2.0(webpack-cli@4.10.0)(webpack@5.90.1) + '@webpack-cli/info': 1.5.0(webpack-cli@4.10.0) + '@webpack-cli/serve': 1.7.0(webpack-cli@4.10.0) colorette: 2.0.20 commander: 7.2.0 cross-spawn: 7.0.3 @@ -6392,7 +6400,7 @@ snapshots: neo-async: 2.6.2 schema-utils: 3.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(webpack@5.90.1(webpack-cli@4.10.0)) + terser-webpack-plugin: 5.3.10(webpack@5.90.1) watchpack: 2.4.0 webpack-sources: 3.2.3 optionalDependencies: diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js index eed62f98..cec94175 100644 --- a/src/api/Logs/index.js +++ b/src/api/Logs/index.js @@ -6,9 +6,9 @@ import { DEFAULT_SERVICE, MAX_LOG_LIMIT, } from "./../Utils/constants"; +import IntervalTree from "@flatten-js/interval-tree"; -const MAX_FETCH_LOGS = 200000; -let logsDataGlobal = []; +const MAX_FETCH_LOGS = 20000; /** * Tranform log from the format received from the API to the format @@ -72,7 +72,10 @@ function getRequestString(label, value) { } function getRequestDate(label, value) { - return getRequestString(label, (value ?? 0) / 1000); + return getRequestString( + label, + value ? new Number(value / 1000).toFixed(0) : 0, + ); } /** @@ -82,7 +85,7 @@ function getRequestDate(label, value) { */ function getLogsParam(queryParam = {}) { return [ - getRequestString("limit", queryParam?.limit || MAX_LOG_LIMIT), + getRequestString("limit", queryParam?.limit), getRequestList("level", queryParam.level?.selected), getRequestList("services", queryParam.service?.selected), getRequestList("tags", queryParam.tag?.selected), @@ -95,28 +98,28 @@ function getLogsParam(queryParam = {}) { .join("&"); } -async function getLogs(maximum = 0) { +async function getLogs(fromDate, toDate, limit) { + // console.log("GETLOGS", fromDate, toDate, limit); const path = "v1/logs/?" + getLogsParam({ - limit: MAX_FETCH_LOGS, + limit, date: { - from: logsDataGlobal.length ? logsDataGlobal[0].timestamp : null, - to: null, + from: fromDate, + to: toDate, }, }); const response = await Rest.get({ path }); const data = response?.data || []; - const oldLogs = logsDataGlobal || []; const newLogs = data.map(transformLog); - - return (logsDataGlobal = ( - Features.get("noLogsDedupe") - ? newLogs.concat(oldLogs) - : logsDedupe(oldLogs, newLogs) - ).slice(-maximum)); + return newLogs; + // return ( + // Features.get("noLogsDedupe") + // ? newLogs.concat(oldLogs) + // : logsDedupe(oldLogs, newLogs) + // ).slice(-maximum); } function noSelection(obj) { @@ -172,16 +175,28 @@ export default class Logs { singleton = this; this.subs = new Map(); + this.maximum = MAX_FETCH_LOGS; + this.lastInterval = []; + this.fetchingAbsent = false; this.refresh(); - this.maximum = maximum; } clear() { - logsDataGlobal = []; + this.tree = new IntervalTree(); this.update(); } - refresh() { + getLastTo() { + return this.lastInterval.length + ? this.lastInterval[this.lastInterval.length - 1].timestamp + : null; + } + + getLastFrom() { + return this.lastInterval.length ? this.lastInterval[0].timestamp : null; + } + + async refresh() { this.streaming = Features.get("logStreaming"); this.clear(); @@ -190,18 +205,22 @@ export default class Logs { const sock = webSocketOpen({ path: "/ws/logs", params }); sock.onmessage = (msg) => { const item = websocketTransform(JSON.parse(msg?.data ?? {})); - logsDataGlobal = [transformLog(item, 0, [item], 0.000001)] - .concat(logsDataGlobal) - .slice(-maximum); + + this.pushInterval([transformLog(item, 0, [item], 0.000001)]); + this.update(); }; - getLogs(this.maximum).then(() => this.update()); + + this.pushInterval(await getLogs(this.getLastFrom(), null, this.maximum)); + + this.update(); } else this.getLogs(); } async getLogs() { try { - await getLogs(this.maximum); + this.pushInterval(await getLogs(this.getLastFrom(), null, this.maximum)); + this.update(); } catch (e) { console.log("Failed getting logs", e); @@ -210,7 +229,123 @@ export default class Logs { } get() { - return logsDataGlobal; + let total = []; + + for (const innerInterval of this.tree.iterate()) + total = innerInterval.concat(total); + + return total; + } + + getAbsentIntervals(fromTime, toTime = this.lastInterval[0]?.timestamp) { + if (!fromTime) return []; + + let absentStart = fromTime; + const absentTimes = []; + + for (const innerKey of this.tree.iterate( + [fromTime, toTime], + (_value, key) => key, + )) { + const { low, high } = innerKey; + + if (absentStart < low) absentTimes.push([absentStart, low]); + + absentStart = high; + } + + if (absentStart < toTime) absentTimes.push([absentStart, toTime]); + + return absentTimes; + } + + getKey(interval) { + return [interval[interval.length - 1].timestamp, interval[0].timestamp]; + } + + pushInterval(interval) { + if (this.lastInterval.length) + this.tree.remove(this.getKey(this.lastInterval), this.lastInterval); + + this.lastInterval = interval.concat(this.lastInterval).slice(-this.maximum); + + this.tree.insert(this.getKey(this.lastInterval), this.lastInterval); + } + + getFormattedKey(intervalKey) { + const [start, end] = intervalKey; + return [new Date(start).toISOString(), new Date(end).toISOString()]; + } + + putInterval(intervalKey, interval) { + if (!interval.length) { + this.tree.insert(intervalKey, []); + return; + } + + const [fromDate, toDate] = intervalKey; + + const trueKey = this.getKey(interval); + + const [trueFromDate, trueToDate] = intervalKey; + + // console.log("putInterval", this.getFormattedKey(trueKey), this.getFormattedKey(intervalKey), intervalKey, interval, this.tree); + + for (const innerInterval of this.tree.iterate(trueKey)) { + const innerKey = this.getKey(innerInterval); + const [innerFromDate, innerToDate] = innerKey; + + if (innerFromDate === toDate) { + this.tree.remove(innerKey); + // console.log("putting inner ", this.getFormattedKey(innerKey)[1], "on top of", this.getFormattedKey(trueKey)[0]); + this.tree.insert( + [trueFromDate, innerToDate], + interval.concat(innerInterval), + ); + return; + } + + if (innerToDate === fromDate) { + this.tree.remove(innerKey); + // console.log("putting", this.getFormattedKey(trueKey)[1], "on top of inner", this.getFormattedKey(innerKey)[0]); + this.tree.insert( + [innerFromDate, trueToDate], + innerInterval.concat(interval), + ); + return; + } + } + + this.tree.insert(trueKey, interval); + } + + async fetchAbsent(selectedFromDate, selectedToDate) { + if (this.fetchingAbsent) return; + + const fromDate = selectedFromDate ? selectedFromDate.getTime() : null; + const toDate = + (selectedToDate ? selectedToDate.getTime() : null) || this.getLastTo(); + + const absentIntervals = this.getAbsentIntervals(fromDate, toDate); + + if (!absentIntervals.length) return; + + this.fetchingAbsent = true; + + // console.log("ABSENT!", absentIntervals); + + const allAbsent = await Promise.all( + absentIntervals.map((interval) => + getLogs(interval[0], interval[1]).then((logs) => [interval, logs]), + ), + ); + + // console.log("allAbsent are", allAbsent); + for (const [key, absent] of allAbsent) this.putInterval(key, absent); + + if (allAbsent.length) this.update(); + + this.fetchingAbsent = false; } filter(query = {}) { @@ -224,7 +359,9 @@ export default class Logs { message = "", } = query; - return logsDataGlobal.filter( + this.fetchAbsent(selectedFromDate, selectedToDate); + + return this.get().filter( (item) => (levels[item.level] || noSelection(levels)) && (service[item.service] || noSelection(service)) && @@ -242,7 +379,7 @@ export default class Logs { } update() { - for (const [sub] of this.subs) sub(logsDataGlobal); + for (const [sub] of this.subs) sub(this.get()); } } From c9bf4ace675b5d866177faab4478f5a064b888ec Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Tue, 10 Dec 2024 10:39:28 +0000 Subject: [PATCH 3/8] unshift logs when combining websockets with rest request --- src/api/Logs/index.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js index cec94175..2a129651 100644 --- a/src/api/Logs/index.js +++ b/src/api/Logs/index.js @@ -211,7 +211,7 @@ export default class Logs { this.update(); }; - this.pushInterval(await getLogs(this.getLastFrom(), null, this.maximum)); + this.shiftInterval(await getLogs(this.getLastFrom(), null, this.maximum)); this.update(); } else this.getLogs(); @@ -263,15 +263,23 @@ export default class Logs { return [interval[interval.length - 1].timestamp, interval[0].timestamp]; } - pushInterval(interval) { + setLastInterval(interval) { if (this.lastInterval.length) this.tree.remove(this.getKey(this.lastInterval), this.lastInterval); - this.lastInterval = interval.concat(this.lastInterval).slice(-this.maximum); + this.lastInterval = interval.slice(-this.maximum); this.tree.insert(this.getKey(this.lastInterval), this.lastInterval); } + shiftInterval(interval) { + this.setLastInterval(this.lastInterval.concat(interval)); + } + + pushInterval(interval) { + this.setLastInterval(interval.concat(this.lastInterval)); + } + getFormattedKey(intervalKey) { const [start, end] = intervalKey; return [new Date(start).toISOString(), new Date(end).toISOString()]; From e389e825202e72e503bffc8ef56c2b631b98b106 Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Wed, 11 Dec 2024 08:34:36 +0000 Subject: [PATCH 4/8] Add a subscriptionTree to ensure we discard unused logs --- src/api/Logs/index.js | 96 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 11 deletions(-) diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js index 2a129651..38ee98cc 100644 --- a/src/api/Logs/index.js +++ b/src/api/Logs/index.js @@ -9,6 +9,7 @@ import { import IntervalTree from "@flatten-js/interval-tree"; const MAX_FETCH_LOGS = 20000; +const END_TIMES = 8640000000000000; /** * Tranform log from the format received from the API to the format @@ -178,6 +179,7 @@ export default class Logs { this.maximum = MAX_FETCH_LOGS; this.lastInterval = []; this.fetchingAbsent = false; + this.subscriptionTree = new IntervalTree(); this.refresh(); } @@ -292,20 +294,15 @@ export default class Logs { } const [fromDate, toDate] = intervalKey; - const trueKey = this.getKey(interval); - const [trueFromDate, trueToDate] = intervalKey; - // console.log("putInterval", this.getFormattedKey(trueKey), this.getFormattedKey(intervalKey), intervalKey, interval, this.tree); - for (const innerInterval of this.tree.iterate(trueKey)) { const innerKey = this.getKey(innerInterval); const [innerFromDate, innerToDate] = innerKey; if (innerFromDate === toDate) { this.tree.remove(innerKey); - // console.log("putting inner ", this.getFormattedKey(innerKey)[1], "on top of", this.getFormattedKey(trueKey)[0]); this.tree.insert( [trueFromDate, innerToDate], interval.concat(innerInterval), @@ -315,7 +312,6 @@ export default class Logs { if (innerToDate === fromDate) { this.tree.remove(innerKey); - // console.log("putting", this.getFormattedKey(trueKey)[1], "on top of inner", this.getFormattedKey(innerKey)[0]); this.tree.insert( [innerFromDate, trueToDate], innerInterval.concat(interval), @@ -340,15 +336,12 @@ export default class Logs { this.fetchingAbsent = true; - // console.log("ABSENT!", absentIntervals); - const allAbsent = await Promise.all( absentIntervals.map((interval) => getLogs(interval[0], interval[1]).then((logs) => [interval, logs]), ), ); - // console.log("allAbsent are", allAbsent); for (const [key, absent] of allAbsent) this.putInterval(key, absent); if (allAbsent.length) this.update(); @@ -367,8 +360,6 @@ export default class Logs { message = "", } = query; - this.fetchAbsent(selectedFromDate, selectedToDate); - return this.get().filter( (item) => (levels[item.level] || noSelection(levels)) && @@ -386,6 +377,89 @@ export default class Logs { return () => this.subs.delete(callback); } + putSubscriptionInterval(intervalKey) { + let [fromDate, toDate] = intervalKey; + const matches = []; + + for (const [value, key] of this.subscriptionTree.iterate( + intervalKey, + (value, key) => [value, key], + )) { + this.subscriptionTree.remove([key.low, key.high], value); + + if (key.low < fromDate) { + if (fromDate < key.high) { + matches.push([key.low, fromDate, value]); + matches.push([fromDate, key.high, value + 1]); + } else if (fromDate === key.high) + matches.push([key.low, fromDate, value + 1]); + // else - fromDate > key.high shouldn't happen + } else if (key.low === fromDate) { + if (toDate < key.high) { + matches.push([key.low, toDate, value + 1]); + matches.push([toDate, key.high, value]); + } else if (toDate === key.high) + matches.push([key.low, toDate, value + 1]); + // else - toDate > key.high shouldn't happen + } else if (key.low > fromDate) { + if (toDate < key.high) { + matches.push([key.low, toDate, value + 1]); + matches.push([toDate, key.high, value]); + } else { + matches.push([key.low, key.high, value + 1]); + } + } + } + + if (!matches.length) + return this.subscriptionTree.insert([fromDate, toDate], 1); + + for (const [innerFrom, innerTo, value] of matches) + this.subscriptionTree.insert([innerFrom, innerTo], value); + + let lastTo = fromDate; + for (const [innerFrom, innerTo] of matches) { + if (innerFrom > lastTo) + this.subscriptionTree.insert([lastTo, innerFrom], 1); + lastTo = innerTo; + } + + if (toDate > lastTo) this.subscriptionTree.insert([lastTo, toDate], 1); + } + + delSubscriptionInterval(intervalKey) { + for (const [value, key] of this.subscriptionTree.iterate( + intervalKey, + (value, key) => [value, key], + )) { + const innerKey = [key.low, key.high]; + this.subscriptionTree.remove(innerKey, value); + if (value - 1 > 0) this.subscriptionTree.insert(innerKey, value - 1); + } + } + + subscribe(callback, outerQuery = {}) { + const { selectedFromDate, selectedToDate } = outerQuery; + + this.fetchAbsent(selectedFromDate, selectedToDate); + + const key = + selectedFromDate || selectedToDate + ? [ + selectedFromDate ? selectedFromDate.getTime() : -END_TIMES, + (selectedToDate ? selectedToDate.getTime() : null) || END_TIMES, + ] + : null; + + if (key) this.putSubscriptionInterval(key); + + this.subs.set(callback, true); + return () => { + if (key) this.delSubscriptionInterval(key); + this.subs.delete(callback); + }; + } + update() { for (const [sub] of this.subs) sub(this.get()); } From af2807eea31be75485bede08fa4712cd2fed53d2 Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Thu, 12 Dec 2024 11:02:05 +0000 Subject: [PATCH 5/8] get rid of unused logs --- src/api/Logs/index.js | 183 +++++++++++++++++++++++++----------------- 1 file changed, 110 insertions(+), 73 deletions(-) diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js index 38ee98cc..4f66f22a 100644 --- a/src/api/Logs/index.js +++ b/src/api/Logs/index.js @@ -86,7 +86,7 @@ function getRequestDate(label, value) { */ function getLogsParam(queryParam = {}) { return [ - getRequestString("limit", queryParam?.limit), + getRequestString("limit", queryParam.limit), getRequestList("level", queryParam.level?.selected), getRequestList("services", queryParam.service?.selected), getRequestList("tags", queryParam.tag?.selected), @@ -100,21 +100,20 @@ function getLogsParam(queryParam = {}) { } async function getLogs(fromDate, toDate, limit) { - // console.log("GETLOGS", fromDate, toDate, limit); - const path = - "v1/logs/?" + - getLogsParam({ - limit, - date: { - from: fromDate, - to: toDate, - }, - }); + const paramObj = { + limit: limit || MAX_FETCH_LOGS, + date: { + from: fromDate, + to: toDate, + }, + }; + const path = "v1/logs/?" + getLogsParam(paramObj); const response = await Rest.get({ path }); const data = response?.data || []; const newLogs = data.map(transformLog); + // console.log("GETLOGS", newLogs?.[newLogs.length - 1]?.timestamp, newLogs?.[0]?.timestamp, newLogs, paramObj); return newLogs; // return ( // Features.get("noLogsDedupe") @@ -176,9 +175,10 @@ export default class Logs { singleton = this; this.subs = new Map(); - this.maximum = MAX_FETCH_LOGS; + this.maximum = maximum; this.lastInterval = []; this.fetchingAbsent = false; + this.beginning = -END_TIMES; this.subscriptionTree = new IntervalTree(); this.refresh(); } @@ -188,14 +188,12 @@ export default class Logs { this.update(); } - getLastTo() { - return this.lastInterval.length - ? this.lastInterval[this.lastInterval.length - 1].timestamp - : null; + getLastFrom() { + return this.lastIntervalKey?.[0] ?? null; } - getLastFrom() { - return this.lastInterval.length ? this.lastInterval[0].timestamp : null; + getLastTo() { + return this.lastIntervalKey?.[1] ?? null; } async refresh() { @@ -208,12 +206,12 @@ export default class Logs { sock.onmessage = (msg) => { const item = websocketTransform(JSON.parse(msg?.data ?? {})); - this.pushInterval([transformLog(item, 0, [item], 0.000001)]); + this.pushInterval([transformLog(item, 0, [item], 0.000001)], true); this.update(); }; - this.shiftInterval(await getLogs(this.getLastFrom(), null, this.maximum)); + this.shiftInterval(await getLogs(this.getLastTo(), null, this.maximum)); this.update(); } else this.getLogs(); @@ -221,7 +219,7 @@ export default class Logs { async getLogs() { try { - this.pushInterval(await getLogs(this.getLastFrom(), null, this.maximum)); + this.pushInterval(await getLogs(this.getLastTo(), null, this.maximum)); this.update(); } catch (e) { @@ -232,6 +230,7 @@ export default class Logs { get() { let total = []; + if (!this.lastInterval.length) return []; for (const innerInterval of this.tree.iterate()) total = innerInterval.concat(total); @@ -239,7 +238,7 @@ export default class Logs { return total; } - getAbsentIntervals(fromTime, toTime = this.lastInterval[0]?.timestamp) { + getAbsentIntervals(fromTime, toTime = this.lastIntervalKey?.[1]) { if (!fromTime) return []; let absentStart = fromTime; @@ -265,13 +264,13 @@ export default class Logs { return [interval[interval.length - 1].timestamp, interval[0].timestamp]; } - setLastInterval(interval) { + setLastInterval(interval, intervalKey) { if (this.lastInterval.length) - this.tree.remove(this.getKey(this.lastInterval), this.lastInterval); - - this.lastInterval = interval.slice(-this.maximum); + this.tree.remove(this.lastIntervalKey, this.lastInterval); - this.tree.insert(this.getKey(this.lastInterval), this.lastInterval); + this.lastInterval = interval; + this.lastIntervalKey = intervalKey ?? this.getKey(interval); + this.tree.insert(this.lastIntervalKey, interval); } shiftInterval(interval) { @@ -287,40 +286,33 @@ export default class Logs { return [new Date(start).toISOString(), new Date(end).toISOString()]; } - putInterval(intervalKey, interval) { - if (!interval.length) { - this.tree.insert(intervalKey, []); - return; - } - + // assumes the interval is really absent + insertAbsent(intervalKey, interval) { const [fromDate, toDate] = intervalKey; - const trueKey = this.getKey(interval); - const [trueFromDate, trueToDate] = intervalKey; - for (const innerInterval of this.tree.iterate(trueKey)) { - const innerKey = this.getKey(innerInterval); - const [innerFromDate, innerToDate] = innerKey; + for (const [innerInterval, key] of this.tree.iterate( + intervalKey, + (value, key) => [value, key], + )) { + this.tree.remove(key); - if (innerFromDate === toDate) { - this.tree.remove(innerKey); - this.tree.insert( - [trueFromDate, innerToDate], + if (key.high === fromDate) + return this.tree.insert( + [key.low, toDate], interval.concat(innerInterval), ); - return; - } - if (innerToDate === fromDate) { - this.tree.remove(innerKey); - this.tree.insert( - [innerFromDate, trueToDate], - innerInterval.concat(interval), - ); - return; + if (key.low === toDate) { + const possiblyLastKey = [fromDate, key.high]; + const possiblyLast = innerInterval.concat(interval); + if (innerInterval === this.lastInterval) + return this.setLastInterval(possiblyLast, possiblyLastKey, false); + return this.tree.insert(possiblyLastKey, possiblyLast); } } - this.tree.insert(trueKey, interval); + if (intervalKey[1] === this.lastIntervalKey[1]) + this.tree.insert(intervalKey, interval); } async fetchAbsent(selectedFromDate, selectedToDate) { @@ -328,7 +320,7 @@ export default class Logs { const fromDate = selectedFromDate ? selectedFromDate.getTime() : null; const toDate = - (selectedToDate ? selectedToDate.getTime() : null) || this.getLastTo(); + (selectedToDate ? selectedToDate.getTime() : null) || this.getLastFrom(); const absentIntervals = this.getAbsentIntervals(fromDate, toDate); @@ -342,7 +334,9 @@ export default class Logs { ), ); - for (const [key, absent] of allAbsent) this.putInterval(key, absent); + // console.log("absent", fromDate, toDate, absentIntervals, allAbsent); + + for (const [key, absent] of allAbsent) this.insertAbsent(key, absent); if (allAbsent.length) this.update(); @@ -372,11 +366,6 @@ export default class Logs { ); } - subscribe(callback) { - this.subs.set(callback, true); - return () => this.subs.delete(callback); - } - putSubscriptionInterval(intervalKey) { let [fromDate, toDate] = intervalKey; const matches = []; @@ -385,7 +374,7 @@ export default class Logs { intervalKey, (value, key) => [value, key], )) { - this.subscriptionTree.remove([key.low, key.high], value); + this.subscriptionTree.remove(key, value); if (key.low < fromDate) { if (fromDate < key.high) { @@ -427,14 +416,53 @@ export default class Logs { if (toDate > lastTo) this.subscriptionTree.insert([lastTo, toDate], 1); } + delIntervals(range) { + let last, + trim = false; + + for (const [value, key] of this.tree.iterate(range, (value, key) => [ + value, + key, + ])) { + if (!this.subscriptionTree.intersect_any(key)) + this.tree.remove(key, value); + last = [[key.low, key.high], value]; + trim = true; + } + + if (!this.tree.isEmpty()) + for (const [value, key] of this.tree.iterate(undefined, (value, key) => [ + value, + key, + ])) + last = [[key.low, key.high], value]; + + if (!last) return; + + let [key, value] = last; + + if (key[1] !== this.lastIntervalKey[1]) return; + + if ( + trim && + value.length > MAX_FETCH_LOGS && + !this.subscriptionTree.intersect_any(key) + ) { + value = value.slice(0, MAX_FETCH_LOGS); + key[0] = value[value.length - 1].timestamp; + } + + this.setLastInterval(value, key); + } + delSubscriptionInterval(intervalKey) { for (const [value, key] of this.subscriptionTree.iterate( intervalKey, (value, key) => [value, key], )) { - const innerKey = [key.low, key.high]; - this.subscriptionTree.remove(innerKey, value); - if (value - 1 > 0) this.subscriptionTree.insert(innerKey, value - 1); + this.subscriptionTree.remove(key, value); + if (value - 1 > 0) this.subscriptionTree.insert(key, value - 1); + else setTimeout(() => this.delIntervals([key.low, key.high]), 0); } } @@ -443,25 +471,34 @@ export default class Logs { this.fetchAbsent(selectedFromDate, selectedToDate); - const key = - selectedFromDate || selectedToDate - ? [ - selectedFromDate ? selectedFromDate.getTime() : -END_TIMES, - (selectedToDate ? selectedToDate.getTime() : null) || END_TIMES, - ] - : null; + const key = [ + selectedFromDate ? selectedFromDate.getTime() : this.beginning, + (selectedToDate ? selectedToDate.getTime() : null) || END_TIMES, + ]; - if (key) this.putSubscriptionInterval(key); + this.putSubscriptionInterval(key); this.subs.set(callback, true); return () => { - if (key) this.delSubscriptionInterval(key); + this.delSubscriptionInterval(key); this.subs.delete(callback); }; } update() { - for (const [sub] of this.subs) sub(this.get()); + const logs = this.get(); + + if (logs.length) this.beginning = logs[logs.length - 1].timestamp; + + if ( + this.beginning !== -END_TIMES && + this.subscriptionTree.exist([-END_TIMES, END_TIMES], 1) + ) { + this.subscriptionTree.remove([-END_TIMES, END_TIMES], 1); + this.subscriptionTree.insert([this.beginning, END_TIMES], 1); + } + + for (const [sub] of this.subs) sub(logs); } } From 57c41858da49e72db116400f353b34b0bac8eb98 Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Thu, 12 Dec 2024 14:21:13 +0000 Subject: [PATCH 6/8] Fix sonar --- src/api/Logs/index.js | 92 +++++++++----------------------------- src/api/Utils/constants.js | 1 - 2 files changed, 21 insertions(+), 72 deletions(-) diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js index 4f66f22a..37b81e30 100644 --- a/src/api/Logs/index.js +++ b/src/api/Logs/index.js @@ -1,11 +1,7 @@ import { webSocketOpen } from "../WebSocket"; import Features from "./../Features"; import Rest from "../Rest/Rest"; -import { - DEFAULT_LEVELS, - DEFAULT_SERVICE, - MAX_LOG_LIMIT, -} from "./../Utils/constants"; +import { DEFAULT_LEVELS, DEFAULT_SERVICE } from "./../Utils/constants"; import IntervalTree from "@flatten-js/interval-tree"; const MAX_FETCH_LOGS = 20000; @@ -28,40 +24,6 @@ function transformLog(log, _index, _data, ts_multiplier = 1000) { }; } -/** - * Remove duplicates from logs for the second overlaping the - * current and the last request - * @returns {array} Concatenated logs without duplicates - */ -export function logsDedupe(oldLogs, data) { - if (!data.length) return oldLogs; - - // date of the oldest log received in the current request - const oldDate = data[data.length - 1].timestamp; - // map to store the old logs of the overlaped second - let map = {}; - - // iter over old logs with last timestamp of the new logs - // and put in a map - for (let i = 0; i < oldLogs.length && oldLogs[i].timestamp === oldDate; i++) - map[oldLogs[i].message] = oldLogs[i]; - - // array to store logs from overlap second which had not - // been sent before - let newSecOverlap = []; - let z; - - // iter over new logs (oldest to latest) with last timestamp, - // check if present in last map - // - if not, push - for (z = data.length - 1; z >= 0 && data[z].timestamp === oldDate; z--) - if (!map[data[z].message]) newSecOverlap.push(data[z]); - - // cut new logs up to z, concat with the deduped ones - // and the old logs up to i - return data.slice(0, z + 1).concat(newSecOverlap.reverse(), oldLogs); -} - function getRequestList(label, values) { return values?.length ? label + "=" + values.map((el) => el.label).join() @@ -75,7 +37,7 @@ function getRequestString(label, value) { function getRequestDate(label, value) { return getRequestString( label, - value ? new Number(value / 1000).toFixed(0) : 0, + value ? (new Number(value) / 1000).toFixed(0) : 0, ); } @@ -113,13 +75,7 @@ async function getLogs(fromDate, toDate, limit) { const data = response?.data || []; const newLogs = data.map(transformLog); - // console.log("GETLOGS", newLogs?.[newLogs.length - 1]?.timestamp, newLogs?.[0]?.timestamp, newLogs, paramObj); return newLogs; - // return ( - // Features.get("noLogsDedupe") - // ? newLogs.concat(oldLogs) - // : logsDedupe(oldLogs, newLogs) - // ).slice(-maximum); } function noSelection(obj) { @@ -174,6 +130,10 @@ export default class Logs { if (singleton) return singleton; singleton = this; + this.init(maximum); + } + + init(maximum = 0) { this.subs = new Map(); this.maximum = maximum; this.lastInterval = []; @@ -183,11 +143,6 @@ export default class Logs { this.refresh(); } - clear() { - this.tree = new IntervalTree(); - this.update(); - } - getLastFrom() { return this.lastIntervalKey?.[0] ?? null; } @@ -198,7 +153,8 @@ export default class Logs { async refresh() { this.streaming = Features.get("logStreaming"); - this.clear(); + this.tree = new IntervalTree(); + this.update(); if (this.streaming) { const params = new URLSearchParams(); @@ -334,8 +290,6 @@ export default class Logs { ), ); - // console.log("absent", fromDate, toDate, absentIntervals, allAbsent); - for (const [key, absent] of allAbsent) this.insertAbsent(key, absent); if (allAbsent.length) this.update(); @@ -366,7 +320,7 @@ export default class Logs { ); } - putSubscriptionInterval(intervalKey) { + collectIntersections(intervalKey) { let [fromDate, toDate] = intervalKey; const matches = []; @@ -376,30 +330,26 @@ export default class Logs { )) { this.subscriptionTree.remove(key, value); + // should work well in all expected situations if (key.low < fromDate) { if (fromDate < key.high) { matches.push([key.low, fromDate, value]); matches.push([fromDate, key.high, value + 1]); } else if (fromDate === key.high) matches.push([key.low, fromDate, value + 1]); - // else - fromDate > key.high shouldn't happen - } else if (key.low === fromDate) { - if (toDate < key.high) { - matches.push([key.low, toDate, value + 1]); - matches.push([toDate, key.high, value]); - } else if (toDate === key.high) - matches.push([key.low, toDate, value + 1]); - // else - toDate > key.high shouldn't happen - } else if (key.low > fromDate) { - if (toDate < key.high) { - matches.push([key.low, toDate, value + 1]); - matches.push([toDate, key.high, value]); - } else { - matches.push([key.low, key.high, value + 1]); - } - } + } else if (toDate < key.high) { + matches.push([key.low, toDate, value + 1]); + matches.push([toDate, key.high, value]); + } else matches.push([key.low, key.high, value + 1]); } + return matches; + } + + putSubscriptionInterval(intervalKey) { + let [fromDate, toDate] = intervalKey; + const matches = this.collectIntersections(intervalKey); + if (!matches.length) return this.subscriptionTree.insert([fromDate, toDate], 1); diff --git a/src/api/Utils/constants.js b/src/api/Utils/constants.js index f1b724bc..0463b5da 100644 --- a/src/api/Utils/constants.js +++ b/src/api/Utils/constants.js @@ -1,4 +1,3 @@ -export const MAX_LOG_LIMIT = 10000; export const GLOBAL_WORKSPACE = "global"; // BroadcastChannel events From 5871c00d6b1e45890940136959bc104bf2972bdb Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Thu, 12 Dec 2024 16:55:36 +0000 Subject: [PATCH 7/8] Better gardener (and trimming) --- src/api/Logs/index.js | 46 +++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js index 37b81e30..04a16a4b 100644 --- a/src/api/Logs/index.js +++ b/src/api/Logs/index.js @@ -7,6 +7,8 @@ import IntervalTree from "@flatten-js/interval-tree"; const MAX_FETCH_LOGS = 20000; const END_TIMES = 8640000000000000; +const WHOLE_TIME = [-END_TIMES, END_TIMES]; + /** * Tranform log from the format received from the API to the format * required to be rendered @@ -61,9 +63,9 @@ function getLogsParam(queryParam = {}) { .join("&"); } -async function getLogs(fromDate, toDate, limit) { +async function getLogs(fromDate, toDate) { const paramObj = { - limit: limit || MAX_FETCH_LOGS, + limit: MAX_FETCH_LOGS, date: { from: fromDate, to: toDate, @@ -126,19 +128,18 @@ export default class Logs { }; // change maximum to another number to limit the amount of kept logs - constructor(maximum = 0) { + constructor() { if (singleton) return singleton; - singleton = this; - this.init(maximum); + this.init(); } - init(maximum = 0) { + init() { this.subs = new Map(); - this.maximum = maximum; this.lastInterval = []; this.fetchingAbsent = false; this.beginning = -END_TIMES; + this.unbound = 0; this.subscriptionTree = new IntervalTree(); this.refresh(); } @@ -167,7 +168,7 @@ export default class Logs { this.update(); }; - this.shiftInterval(await getLogs(this.getLastTo(), null, this.maximum)); + this.shiftInterval(await getLogs(this.getLastTo(), null)); this.update(); } else this.getLogs(); @@ -175,7 +176,7 @@ export default class Logs { async getLogs() { try { - this.pushInterval(await getLogs(this.getLastTo(), null, this.maximum)); + this.pushInterval(await getLogs(this.getLastTo(), null)); this.update(); } catch (e) { @@ -367,8 +368,7 @@ export default class Logs { } delIntervals(range) { - let last, - trim = false; + let last; for (const [value, key] of this.tree.iterate(range, (value, key) => [ value, @@ -377,7 +377,6 @@ export default class Logs { if (!this.subscriptionTree.intersect_any(key)) this.tree.remove(key, value); last = [[key.low, key.high], value]; - trim = true; } if (!this.tree.isEmpty()) @@ -393,12 +392,13 @@ export default class Logs { if (key[1] !== this.lastIntervalKey[1]) return; - if ( - trim && - value.length > MAX_FETCH_LOGS && - !this.subscriptionTree.intersect_any(key) - ) { - value = value.slice(0, MAX_FETCH_LOGS); + if (value.length > MAX_FETCH_LOGS) { + if ( + !this.subscriptionTree.intersect_any(key) || + (this.subscriptionTree.size === 1 && this.unbound === 0) + ) + value = value.slice(0, MAX_FETCH_LOGS); + key[0] = value[value.length - 1].timestamp; } @@ -417,7 +417,9 @@ export default class Logs { } subscribe(callback, outerQuery = {}) { - const { selectedFromDate, selectedToDate } = outerQuery; + const { selectedFromDate, selectedToDate, limit } = outerQuery; + + if (!limit) this.unbound++; this.fetchAbsent(selectedFromDate, selectedToDate); @@ -430,6 +432,8 @@ export default class Logs { this.subs.set(callback, true); return () => { + if (!limit) this.unbound--; + this.delSubscriptionInterval(key); this.subs.delete(callback); }; @@ -442,9 +446,9 @@ export default class Logs { if ( this.beginning !== -END_TIMES && - this.subscriptionTree.exist([-END_TIMES, END_TIMES], 1) + this.subscriptionTree.exist(WHOLE_TIME, 1) ) { - this.subscriptionTree.remove([-END_TIMES, END_TIMES], 1); + this.subscriptionTree.remove(WHOLE_TIME, 1); this.subscriptionTree.insert([this.beginning, END_TIMES], 1); } From 595d943a795e80868016a863245e679a382553ce Mon Sep 17 00:00:00 2001 From: Paulo Andre Azevedo Quirino Date: Thu, 9 Jan 2025 09:20:17 +0000 Subject: [PATCH 8/8] Use @tty-pt/logs --- package.json | 1 + pnpm-lock.yaml | 19 ++ src/api/Logs/index.js | 434 +++--------------------------------------- 3 files changed, 50 insertions(+), 404 deletions(-) diff --git a/package.json b/package.json index b1804d37..8dafac10 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "dependencies": { "@babel/runtime": "^7.15.4", "@flatten-js/interval-tree": "^1.1.3", + "@tty-pt/logs": "^0.0.26", "fast-equals": "^5.0.1", "js-yaml": "^4.1.0", "jwt-decode": "^3.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8763ce22..67b13fcb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@flatten-js/interval-tree': specifier: ^1.1.3 version: 1.1.3 + '@tty-pt/logs': + specifier: ^0.0.26 + version: 0.0.26(@flatten-js/interval-tree@1.1.3)(bintrees@1.0.2) fast-equals: specifier: ^5.0.1 version: 5.0.1 @@ -925,6 +928,12 @@ packages: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} + '@tty-pt/logs@0.0.26': + resolution: {integrity: sha512-cJ/GNqXKrkXsnr6Al9kKC9fmbXpXdYTZ91t+9ptW3PEsvmq0PUpOhW2RZvovSoo/3Rall1dQ2iUiD+/enZ1GIg==} + peerDependencies: + '@flatten-js/interval-tree': ^1.1.3 + bintrees: ^1.0.2 + '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} @@ -1217,6 +1226,9 @@ packages: big.js@5.2.2: resolution: {integrity: sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + body-parser@1.20.1: resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -4130,6 +4142,11 @@ snapshots: '@tootallnate/once@2.0.0': {} + '@tty-pt/logs@0.0.26(@flatten-js/interval-tree@1.1.3)(bintrees@1.0.2)': + dependencies: + '@flatten-js/interval-tree': 1.1.3 + bintrees: 1.0.2 + '@types/babel__core@7.20.5': dependencies: '@babel/parser': 7.23.9 @@ -4483,6 +4500,8 @@ snapshots: big.js@5.2.2: {} + bintrees@1.0.2: {} + body-parser@1.20.1: dependencies: bytes: 3.1.2 diff --git a/src/api/Logs/index.js b/src/api/Logs/index.js index 04a16a4b..c31917b9 100644 --- a/src/api/Logs/index.js +++ b/src/api/Logs/index.js @@ -2,20 +2,16 @@ import { webSocketOpen } from "../WebSocket"; import Features from "./../Features"; import Rest from "../Rest/Rest"; import { DEFAULT_LEVELS, DEFAULT_SERVICE } from "./../Utils/constants"; -import IntervalTree from "@flatten-js/interval-tree"; - -const MAX_FETCH_LOGS = 20000; -const END_TIMES = 8640000000000000; - -const WHOLE_TIME = [-END_TIMES, END_TIMES]; +import Logs from "@tty-pt/logs"; /** * Tranform log from the format received from the API to the format * required to be rendered * @returns {array} Transformed log */ -function transformLog(log, _index, _data, ts_multiplier = 1000) { - const timestamp = ts_multiplier * log.time; +function transform(log, _index, _array, isStream) { + const multiplier = isStream ? 0.000001 : 1000; + const timestamp = multiplier * log.time; const date = new Date(timestamp); return { ...log, @@ -26,75 +22,8 @@ function transformLog(log, _index, _data, ts_multiplier = 1000) { }; } -function getRequestList(label, values) { - return values?.length - ? label + "=" + values.map((el) => el.label).join() - : ""; -} - -function getRequestString(label, value) { - return value ? label + "=" + value : ""; -} - -function getRequestDate(label, value) { - return getRequestString( - label, - value ? (new Number(value) / 1000).toFixed(0) : 0, - ); -} - -/** - * Get Logs parameter string - * @param {LogQueryParam} queryParam : Object to construct query string - * @returns {string} query parameter string - */ -function getLogsParam(queryParam = {}) { - return [ - getRequestString("limit", queryParam.limit), - getRequestList("level", queryParam.level?.selected), - getRequestList("services", queryParam.service?.selected), - getRequestList("tags", queryParam.tag?.selected), - getRequestString("message", queryParam.searchMessage), - getRequestDate("fromDate", queryParam.date?.from), - getRequestDate("toDate", queryParam.date?.to), - getRequestList("robots", queryParam.robot?.selected), - ] - .filter((a) => !!a) - .join("&"); -} - -async function getLogs(fromDate, toDate) { - const paramObj = { - limit: MAX_FETCH_LOGS, - date: { - from: fromDate, - to: toDate, - }, - }; - const path = "v1/logs/?" + getLogsParam(paramObj); - - const response = await Rest.get({ path }); - - const data = response?.data || []; - const newLogs = data.map(transformLog); - return newLogs; -} - -function noSelection(obj) { - for (let key in obj) if (obj[key]) return false; - return true; -} - -function matchTags(tags, item) { - for (const tag in tags) - if (item[tag] !== undefined) continue; - else return false; - return true; -} - -let singleton = null; - -function websocketTransform(original) { +function streamTransform(msg) { + const original = JSON.parse(msg?.data ?? {}); const { message } = original; let tags = {}; @@ -121,339 +50,36 @@ function websocketTransform(original) { }; } -export default class Logs { +export default class MovaiLogs extends Logs { static CONSTANTS = { DEFAULT_LEVELS, DEFAULT_SERVICE, }; - // change maximum to another number to limit the amount of kept logs constructor() { - if (singleton) return singleton; - singleton = this; - this.init(); - } - - init() { - this.subs = new Map(); - this.lastInterval = []; - this.fetchingAbsent = false; - this.beginning = -END_TIMES; - this.unbound = 0; - this.subscriptionTree = new IntervalTree(); - this.refresh(); - } - - getLastFrom() { - return this.lastIntervalKey?.[0] ?? null; - } - - getLastTo() { - return this.lastIntervalKey?.[1] ?? null; - } - - async refresh() { - this.streaming = Features.get("logStreaming"); - this.tree = new IntervalTree(); - this.update(); - - if (this.streaming) { - const params = new URLSearchParams(); - const sock = webSocketOpen({ path: "/ws/logs", params }); - sock.onmessage = (msg) => { - const item = websocketTransform(JSON.parse(msg?.data ?? {})); - - this.pushInterval([transformLog(item, 0, [item], 0.000001)], true); - - this.update(); - }; - - this.shiftInterval(await getLogs(this.getLastTo(), null)); - - this.update(); - } else this.getLogs(); - } - - async getLogs() { - try { - this.pushInterval(await getLogs(this.getLastTo(), null)); - - this.update(); - } catch (e) { - console.log("Failed getting logs", e); - } - setTimeout(() => this.getLogs(), 3000); - } - - get() { - let total = []; - if (!this.lastInterval.length) return []; - - for (const innerInterval of this.tree.iterate()) - total = innerInterval.concat(total); - - return total; - } - - getAbsentIntervals(fromTime, toTime = this.lastIntervalKey?.[1]) { - if (!fromTime) return []; - - let absentStart = fromTime; - const absentTimes = []; - - for (const innerKey of this.tree.iterate( - [fromTime, toTime], - (_value, key) => key, - )) { - const { low, high } = innerKey; - - if (absentStart < low) absentTimes.push([absentStart, low]); - - absentStart = high; - } - - if (absentStart < toTime) absentTimes.push([absentStart, toTime]); - - return absentTimes; - } - - getKey(interval) { - return [interval[interval.length - 1].timestamp, interval[0].timestamp]; - } - - setLastInterval(interval, intervalKey) { - if (this.lastInterval.length) - this.tree.remove(this.lastIntervalKey, this.lastInterval); - - this.lastInterval = interval; - this.lastIntervalKey = intervalKey ?? this.getKey(interval); - this.tree.insert(this.lastIntervalKey, interval); - } - - shiftInterval(interval) { - this.setLastInterval(this.lastInterval.concat(interval)); - } - - pushInterval(interval) { - this.setLastInterval(interval.concat(this.lastInterval)); - } - - getFormattedKey(intervalKey) { - const [start, end] = intervalKey; - return [new Date(start).toISOString(), new Date(end).toISOString()]; - } - - // assumes the interval is really absent - insertAbsent(intervalKey, interval) { - const [fromDate, toDate] = intervalKey; - - for (const [innerInterval, key] of this.tree.iterate( - intervalKey, - (value, key) => [value, key], - )) { - this.tree.remove(key); - - if (key.high === fromDate) - return this.tree.insert( - [key.low, toDate], - interval.concat(innerInterval), - ); - - if (key.low === toDate) { - const possiblyLastKey = [fromDate, key.high]; - const possiblyLast = innerInterval.concat(interval); - if (innerInterval === this.lastInterval) - return this.setLastInterval(possiblyLast, possiblyLastKey, false); - return this.tree.insert(possiblyLastKey, possiblyLast); - } - } - - if (intervalKey[1] === this.lastIntervalKey[1]) - this.tree.insert(intervalKey, interval); - } - - async fetchAbsent(selectedFromDate, selectedToDate) { - if (this.fetchingAbsent) return; - - const fromDate = selectedFromDate ? selectedFromDate.getTime() : null; - const toDate = - (selectedToDate ? selectedToDate.getTime() : null) || this.getLastFrom(); - - const absentIntervals = this.getAbsentIntervals(fromDate, toDate); - - if (!absentIntervals.length) return; - - this.fetchingAbsent = true; - - const allAbsent = await Promise.all( - absentIntervals.map((interval) => - getLogs(interval[0], interval[1]).then((logs) => [interval, logs]), - ), - ); - - for (const [key, absent] of allAbsent) this.insertAbsent(key, absent); - - if (allAbsent.length) this.update(); - - this.fetchingAbsent = false; - } - - filter(query = {}) { - const { - levels = DEFAULT_LEVELS, - service = DEFAULT_SERVICE, - tags = {}, - robots = {}, - selectedFromDate = null, - selectedToDate = null, - message = "", - } = query; - - return this.get().filter( - (item) => - (levels[item.level] || noSelection(levels)) && - (service[item.service] || noSelection(service)) && - (matchTags(tags, item) || noSelection(tags)) && - (item.message || "").includes(message) && - (robots[item.robot] || noSelection(robots)) && - (!selectedFromDate || item.timestamp >= selectedFromDate) && - (!selectedToDate || item.timestamp <= selectedToDate), - ); - } - - collectIntersections(intervalKey) { - let [fromDate, toDate] = intervalKey; - const matches = []; - - for (const [value, key] of this.subscriptionTree.iterate( - intervalKey, - (value, key) => [value, key], - )) { - this.subscriptionTree.remove(key, value); - - // should work well in all expected situations - if (key.low < fromDate) { - if (fromDate < key.high) { - matches.push([key.low, fromDate, value]); - matches.push([fromDate, key.high, value + 1]); - } else if (fromDate === key.high) - matches.push([key.low, fromDate, value + 1]); - } else if (toDate < key.high) { - matches.push([key.low, toDate, value + 1]); - matches.push([toDate, key.high, value]); - } else matches.push([key.low, key.high, value + 1]); - } - - return matches; - } - - putSubscriptionInterval(intervalKey) { - let [fromDate, toDate] = intervalKey; - const matches = this.collectIntersections(intervalKey); - - if (!matches.length) - return this.subscriptionTree.insert([fromDate, toDate], 1); - - for (const [innerFrom, innerTo, value] of matches) - this.subscriptionTree.insert([innerFrom, innerTo], value); - - let lastTo = fromDate; - for (const [innerFrom, innerTo] of matches) { - if (innerFrom > lastTo) - this.subscriptionTree.insert([lastTo, innerFrom], 1); - lastTo = innerTo; - } - - if (toDate > lastTo) this.subscriptionTree.insert([lastTo, toDate], 1); - } - - delIntervals(range) { - let last; - - for (const [value, key] of this.tree.iterate(range, (value, key) => [ - value, - key, - ])) { - if (!this.subscriptionTree.intersect_any(key)) - this.tree.remove(key, value); - last = [[key.low, key.high], value]; - } - - if (!this.tree.isEmpty()) - for (const [value, key] of this.tree.iterate(undefined, (value, key) => [ - value, - key, - ])) - last = [[key.low, key.high], value]; - - if (!last) return; - - let [key, value] = last; - - if (key[1] !== this.lastIntervalKey[1]) return; - - if (value.length > MAX_FETCH_LOGS) { - if ( - !this.subscriptionTree.intersect_any(key) || - (this.subscriptionTree.size === 1 && this.unbound === 0) - ) - value = value.slice(0, MAX_FETCH_LOGS); - - key[0] = value[value.length - 1].timestamp; - } - - this.setLastInterval(value, key); - } - - delSubscriptionInterval(intervalKey) { - for (const [value, key] of this.subscriptionTree.iterate( - intervalKey, - (value, key) => [value, key], - )) { - this.subscriptionTree.remove(key, value); - if (value - 1 > 0) this.subscriptionTree.insert(key, value - 1); - else setTimeout(() => this.delIntervals([key.low, key.high]), 0); - } - } - - subscribe(callback, outerQuery = {}) { - const { selectedFromDate, selectedToDate, limit } = outerQuery; - - if (!limit) this.unbound++; - - this.fetchAbsent(selectedFromDate, selectedToDate); - - const key = [ - selectedFromDate ? selectedFromDate.getTime() : this.beginning, - (selectedToDate ? selectedToDate.getTime() : null) || END_TIMES, - ]; - - this.putSubscriptionInterval(key); - - this.subs.set(callback, true); - return () => { - if (!limit) this.unbound--; - - this.delSubscriptionInterval(key); - this.subs.delete(callback); - }; - } - - update() { - const logs = this.get(); - - if (logs.length) this.beginning = logs[logs.length - 1].timestamp; - - if ( - this.beginning !== -END_TIMES && - this.subscriptionTree.exist(WHOLE_TIME, 1) - ) { - this.subscriptionTree.remove(WHOLE_TIME, 1); - this.subscriptionTree.insert([this.beginning, END_TIMES], 1); - } - - for (const [sub] of this.subs) sub(logs); + super({ + url: "v1/logs/", + fetch: async (path) => (await Rest.get({ path }))?.data, + transform, + + streamUrl: "/ws/logs", + stream: Features.get("logStreaming"), + streamOpen: (streamUrl) => + webSocketOpen({ + path: streamUrl, + params: new URLSearchParams(), + }), + streamTransform, + + fields: { + level: { type: "enumeration", default: DEFAULT_LEVELS }, + service: { type: "enumeration", default: DEFAULT_SERVICE }, + tags: { type: "tags" }, + message: { type: "string" }, + robot: { type: "enumeration" }, + }, + }); } } -globalThis.Logs = Logs; +globalThis.Logs = MovaiLogs;