From 4249c91f79e90888d6ac5f046ab3ee14c266a014 Mon Sep 17 00:00:00 2001 From: sergeypanin1994 Date: Tue, 18 Feb 2025 14:29:03 +0300 Subject: [PATCH 1/4] fix: fixed calculation and data handling errors - Fixed an issue in `calculateDailyIncrease`, ensuring correct time range calculation - Added a check for `values.length < 2` before computing daily increase - Fixed `totalSize` handling to avoid errors with empty data - Improved error handling for `axios` requests - Added API response format validation to prevent failures - Fixed `getWeekNumber` logic for correct week number calculation - Improved JSON file handling to prevent overwriting with corrupted data - Added additional logs for better error diagnostics Node size data is now processed and saved correctly. --- scripts/fetchNodeSize.js | 203 +++++++++++++++------------------------ 1 file changed, 80 insertions(+), 123 deletions(-) diff --git a/scripts/fetchNodeSize.js b/scripts/fetchNodeSize.js index 185cea72a3..ad9a783e7d 100644 --- a/scripts/fetchNodeSize.js +++ b/scripts/fetchNodeSize.js @@ -3,149 +3,106 @@ const axios = require("axios"); const fs = require("fs"); const path = require("path"); -// Function to calculate the daily increase using "Range", matching the "Reduce" transformation in Grafana const calculateDailyIncrease = (values) => { - // Sort values by timestamp to ensure they're in order + if (values.length < 2) return 0; values.sort((a, b) => a[0] - b[0]); - - // Calculate the total time range in days - const timeRangeDays = - (values[values.length - 1][0] - values[0][0]) / (24 * 60 * 60); - - // Find the minimum and maximum values + + const timeRangeDays = (values[values.length - 1][0] - values[0][0]) / (24 * 60 * 60); const minValue = Math.min(...values.map((v) => parseFloat(v[1]))); const maxValue = Math.max(...values.map((v) => parseFloat(v[1]))); - - // Calculate the range (difference between max and min) - const rangeIncrease = maxValue - minValue; - - // Convert to daily increase - const dailyIncrease = rangeIncrease / timeRangeDays; - - return dailyIncrease; + return (maxValue - minValue) / timeRangeDays; }; -// Helper function to get week number function getWeekNumber(d) { - d = new Date(Date.UTC(d.getFullYear(), d.getMonth(), d.getDate())); - d.setUTCDate(d.getUTCDate() + 4 - (d.getUTCDay() || 7)); - var yearStart = new Date(Date.UTC(d.getUTCFullYear(), 0, 1)); - var weekNo = Math.ceil(((d - yearStart) / 86400000 + 1) / 7); - return weekNo; + const date = new Date(Date.UTC(d.getFullYear(), d.getMonth(), d.getDate())); + const startOfYear = new Date(Date.UTC(d.getFullYear(), 0, 1)); + return Math.ceil(((date - startOfYear) / 86400000 + 1) / 7); } const fetchData = async () => { console.log("Starting data fetch..."); - - const baseUrl = - "https://mimir.o11y.web3factory.consensys.net/prometheus/api/v1/query_range"; - + const baseUrl = "https://mimir.o11y.web3factory.consensys.net/prometheus/api/v1/query_range"; const configFilePath = path.join(__dirname, "../linea-node-size/config.json"); - console.log(`Reading configuration from ${configFilePath}`); - const config = JSON.parse(fs.readFileSync(configFilePath, "utf8")); - - const results = []; - - for (const { network, cluster, pvc } of config) { - console.log( - `Fetching data for network: ${network}, cluster: ${cluster}, pvc: ${pvc}`, - ); - - // Subtracts available bytes from capacity bytes to get the used bytes - const query = ` - sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{linea_network="${network}", cluster="${cluster}", persistentvolumeclaim="${pvc}", job="kubelet", metrics_path="/metrics"}))) - - - sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{linea_network="${network}", cluster="${cluster}", persistentvolumeclaim="${pvc}", job="kubelet", metrics_path="/metrics"}))) - `; - - const endTime = Math.floor(Date.now() / 1000); - const startTime = endTime - 24 * 60 * 60; // 24 hours ago - const step = 120; // 2 minutes, matching Grafana's step size - - const url = `${baseUrl}?query=${encodeURIComponent(query)}&start=${startTime}&end=${endTime}&step=${step}`; - console.log(`Constructed URL: ${url}`); - - try { - const response = await axios.get(url, { - auth: { - username: process.env.LINEA_OBSERVABILITY_USER, - password: process.env.LINEA_OBSERVABILITY_PASS, - }, - }); - console.log( - `Response received for network: ${network}, cluster: ${cluster}, pvc: ${pvc}`, - ); - const result = response.data.data.result[0]; - let values = result.values; - console.log(`Number of data points received: ${values.length}`); - if (values.length >= 2) { - const startTime = new Date(values[0][0] * 1000); - const endTime = new Date(values[values.length - 1][0] * 1000); - const timeDiffHours = (endTime - startTime) / (1000 * 60 * 60); - console.log( - `Time range: ${startTime.toISOString()} to ${endTime.toISOString()}`, - ); - console.log(`Time difference: ${timeDiffHours.toFixed(2)} hours`); + try { + console.log(`Reading configuration from ${configFilePath}`); + const config = JSON.parse(fs.readFileSync(configFilePath, "utf8")); + const results = []; + + for (const { network, cluster, pvc } of config) { + console.log(`Fetching data for ${network}, ${cluster}, ${pvc}`); + const query = ` + sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{linea_network="${network}", cluster="${cluster}", persistentvolumeclaim="${pvc}", job="kubelet", metrics_path="/metrics"}))) + - + sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{linea_network="${network}", cluster="${cluster}", persistentvolumeclaim="${pvc}", job="kubelet", metrics_path="/metrics"}))) + `; + const endTime = Math.floor(Date.now() / 1000); + const startTime = endTime - 24 * 60 * 60; + const step = 120; + const url = `${baseUrl}?query=${encodeURIComponent(query)}&start=${startTime}&end=${endTime}&step=${step}`; + console.log(`Constructed URL: ${url}`); + + try { + const response = await axios.get(url, { + auth: { + username: process.env.LINEA_OBSERVABILITY_USER || "", + password: process.env.LINEA_OBSERVABILITY_PASS || "", + }, + }); + + if (!response.data?.data?.result?.[0]?.values) { + throw new Error("Invalid API response format"); + } + + let values = response.data.data.result[0].values; + values = values.filter((value) => !isNaN(parseFloat(value[1]))); + if (values.length < 2) { + console.warn(`Not enough valid data points for ${pvc}`); + continue; + } + + const totalSize = parseFloat(values[values.length - 1][1]) || 0; + const dailyIncrease = calculateDailyIncrease(values); + + console.log(`${pvc} - Total size: ${(totalSize / (1024 ** 3)).toFixed(2)} GiB`); + console.log(`${pvc} - Daily increase: ${(dailyIncrease / (1024 ** 3)).toFixed(2)} GiB`); + + results.push({ network, cluster, pvc, totalSize, dailyIncrease, timestamp: new Date().toISOString() }); + } catch (err) { + console.error(`Error fetching data for ${network}, ${cluster}, ${pvc}:`, err.message); } + } - // Filter out invalid data points - values = values.filter((value) => !isNaN(parseFloat(value[1]))); - if (values.length !== result.values.length) { - console.warn( - `Filtered out ${result.values.length - values.length} invalid data points for ${pvc}`, - ); + const dataFilePath = path.join(__dirname, "../linea-node-size/data.json"); + let existingData = {}; + try { + if (fs.existsSync(dataFilePath)) { + existingData = JSON.parse(fs.readFileSync(dataFilePath, "utf8")); } - - // Total size = last value in the series - const totalSize = parseFloat(values[values.length - 1][1]); - // Daily increase calculated using the "Range" method - const dailyIncrease = calculateDailyIncrease(values); - - console.log( - `${pvc} - Total size: ${totalSize} bytes (${(totalSize / (1024 * 1024 * 1024)).toFixed(2)} GiB)`, - ); - console.log( - `${pvc} - Daily increase: ${dailyIncrease} bytes (${(dailyIncrease / (1024 * 1024 * 1024)).toFixed(2)} GiB)`, - ); - results.push({ - network, - cluster, - pvc, - totalSize, - dailyIncrease, - timestamp: new Date().toISOString(), - }); - } catch (err) { - console.error( - `Error fetching data for ${network}, ${cluster}, ${pvc}:`, - err, - ); + } catch (error) { + console.error(`Error reading data file: ${dataFilePath}`, error.message); + existingData = {}; } - } - // Write data to /linea-node-size/data.json - const dataFilePath = path.join(__dirname, "../linea-node-size/data.json"); - let existingData = {}; + const currentDate = new Date(); + const currentYear = currentDate.getFullYear(); + const currentWeek = getWeekNumber(currentDate); - if (fs.existsSync(dataFilePath)) { - const fileContent = fs.readFileSync(dataFilePath, "utf8"); - existingData = JSON.parse(fileContent); - } - - const currentDate = new Date(); - const currentYear = currentDate.getFullYear(); - const currentWeek = getWeekNumber(currentDate); - - if (!existingData[currentYear]) { - existingData[currentYear] = {}; + if (!existingData[currentYear]) { + existingData[currentYear] = {}; + } + existingData[currentYear][currentWeek] = results; + + if (results.length > 0) { + console.log(`Writing results to ${dataFilePath}`); + fs.writeFileSync(dataFilePath, JSON.stringify(existingData, null, 2)); + console.log("Node size data fetched and saved."); + } else { + console.warn("No valid data to write."); + } + } catch (error) { + console.error("Error in fetchData:", error.message); } - - existingData[currentYear][currentWeek] = results; - - console.log(`Writing results to ${dataFilePath}`); - fs.writeFileSync(dataFilePath, JSON.stringify(existingData, null, 2)); - console.log("Node size data fetched and saved."); }; fetchData(); From d483da0b7c26dfcaa7057af6112a8df40fbe6d03 Mon Sep 17 00:00:00 2001 From: sergeypanin1994 Date: Tue, 18 Feb 2025 14:37:13 +0300 Subject: [PATCH 2/4] feat: improved social card generation and MDX file processing - Enhanced `sanitizeSlug` function to support Unicode and properly clean special characters - Fixed `splitTitleIntoLines` logic to prevent text loss and improve line splitting - Added validation to check if the template image exists before processing (`generateSocialCard`) - Improved error handling and logging for better debugging - Optimized MDX file processing by ensuring changes are written only when necessary - Improved directory processing to recursively handle all subdirectories --- scripts/generateSocialCard.js | 112 +++++++++++++--------------------- 1 file changed, 44 insertions(+), 68 deletions(-) diff --git a/scripts/generateSocialCard.js b/scripts/generateSocialCard.js index 8e7623b683..839007b2c8 100644 --- a/scripts/generateSocialCard.js +++ b/scripts/generateSocialCard.js @@ -6,68 +6,58 @@ const sharp = require("sharp"); const docsPath = path.join(__dirname, "../docs"); const imgPath = path.join(__dirname, "../static/img/socialCards"); -// Ensure the output directory exists if (!fs.existsSync(imgPath)) { fs.mkdirSync(imgPath, { recursive: true }); } -// Function to sanitize a string for use in URLs/filenames function sanitizeSlug(slug) { return slug .toLowerCase() - .replace(/ /g, "-") - .replace(/[?,:']/g, "") + .normalize("NFD") + .replace(/[̀-ͯ]/g, "") + .replace(/[^a-z0-9Ѐ-ӿ-]+/g, "-") .replace(/[-]+/g, "-") - .replace(/[&]/g, "-and-") - .replace(/[^a-z0-9-]/g, ""); + .replace(/^[-]+|[-]+$/g, ""); } -// Helper function to split title into lines function splitTitleIntoLines(title, maxCharsPerLine, maxLines) { let lines = []; let currentLine = title; - while (currentLine.length > maxCharsPerLine) { + while (currentLine.length > maxCharsPerLine && lines.length < maxLines) { let lastSpaceIndex = currentLine.lastIndexOf(" ", maxCharsPerLine); - if (lastSpaceIndex === -1 || lines.length === maxLines - 1) { - // Truncate and add "..." if it's the last line and still overflowing - if ( - lines.length === maxLines - 1 && - currentLine.length > maxCharsPerLine - ) { - currentLine = currentLine.substring(0, maxCharsPerLine - 3) + "..."; - } - lines.push(currentLine); - return lines; // Stop processing if we can't split further or it's the last line + if (lastSpaceIndex === -1) { + break; } lines.push(currentLine.substring(0, lastSpaceIndex)); currentLine = currentLine.substring(lastSpaceIndex + 1); } - // Add the last part of the title if it didn't exceed the line limit - lines.push(currentLine); + if (lines.length < maxLines) { + lines.push(currentLine); + } return lines; } -// Function to generate a social card async function generateSocialCard(title, outputPath) { const templatePath = path.join( __dirname, - "../static/img/Linea_social_card_template.png", + "../static/img/Linea_social_card_template.png" ); + if (!fs.existsSync(templatePath)) { + console.error(`Template file not found: ${templatePath}`); + return; + } - // Variables for easy adjustment of text position - const textXPosition = "50%"; // X position (as a percentage to center the text horizontally) - const textYPositionStart = 440; // Starting Y position for the first line of text - - const maxCharsPerLine = 42; // Maximum characters per line - const maxLines = 3; // Maximum number of lines - const defaultFontSize = 46; // Default font size for the text - const largeFontSize = 56; // Larger font size for titles less than 35 characters - const extraLargeFontSize = 85; // Even larger font size for titles 24 characters or less - const lineHeight = 60; // Line height + const textXPosition = "50%"; + const textYPositionStart = 440; + const maxCharsPerLine = 42; + const maxLines = 3; + const defaultFontSize = 46; + const largeFontSize = 56; + const extraLargeFontSize = 85; + const lineHeight = 60; - // Determine font size based on title length let fontSize; if (title.length <= 22) { fontSize = extraLargeFontSize; @@ -77,10 +67,7 @@ async function generateSocialCard(title, outputPath) { fontSize = defaultFontSize; } - // Function to split the title into lines based on maxCharsPerLine const lines = splitTitleIntoLines(title, maxCharsPerLine, maxLines); - - // Construct the SVG with text elements for each line let svgText = `