Feat: backup to csv (#137)

Feat: backup to csv - generate headers of csv from index mapping - transform json data to csv Refs: #23 --------- Signed-off-by: seven <[email protected]>
geek-fun · Nov 20, 2024 · 0675e6e · 0675e6e
1 parent 4f1a76d
commit 0675e6e
Show file tree

Hide file tree

Showing 9 changed files with 159 additions and 79 deletions.
diff --git a/.eslintrc.cjs b/.eslintrc.cjs
@@ -40,5 +40,5 @@ module.exports = {
     'no-console': 'error',
     'no-debugger': process.env === 'development' ? 'warn' : 'error',
   },
-  ignorePatterns: ['dist','src-tauri'],
+  ignorePatterns: ['dist', 'src-tauri', 'coverage'],
 };
diff --git a/jest.config.cjs b/jest.config.cjs
@@ -1,4 +1,3 @@
-/** @type {import('ts-jest').JestConfigWithTsJest} */
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -43,9 +43,9 @@
   "devDependencies": {
     "@tauri-apps/cli": "^1",
     "@types/debug": "^4.1.12",
-    "@types/jest": "^29.5.12",
-    "@types/markdown-it": "^14.1.1",
+    "@types/jest": "^29.5.14",
     "@types/lodash": "^4.17.12",
+    "@types/markdown-it": "^14.1.1",
     "@typescript-eslint/eslint-plugin": "^7.14.1",
     "@typescript-eslint/parser": "^7.14.1",
     "@vicons/antd": "^0.12.0",
@@ -62,7 +62,7 @@
     "naive-ui": "^2.38.2",
     "prettier": "^3.3.2",
     "sass": "^1.77.6",
-    "ts-jest": "^29.1.5",
+    "ts-jest": "^29.2.5",
     "typescript": "^5.5.2",
     "unplugin-auto-import": "^0.17.6",
     "unplugin-vue-components": "^0.27.1",

diff --git a/src/store/backupRestoreStore.ts b/src/store/backupRestoreStore.ts
@@ -55,12 +55,16 @@ export const useBackupRestoreStore = defineStore('backupRestoreStore', {
       const filePath = `${input.backupFolder}/${input.backupFileName}.${input.backupFileType}`;
       let searchAfter: any[] | undefined = undefined;
       let hasMore = true;
+      let appendFile = false;
 
       try {
         this.backupProgress = {
           complete: 0,
           total: (await client.get(`/${input.index}/_count`)).count,
         };
+        const { [input.index]: backupIndexMapping } = await client.get(`/${input.index}/_mapping`);
+        const csvHeaders = buildCsvHeaders(backupIndexMapping);
+        let dataToWrite = input.backupFileType === 'json' ? '' : csvHeaders.join(',') + '\r\n';
 
         while (hasMore) {
           const response = await client.get(
@@ -90,11 +94,13 @@ export const useBackupRestoreStore = defineStore('backupRestoreStore', {
             hasMore = false;
           } else {
             searchAfter = hits[hits.length - 1].sort;
-            const dataToWrite =
+            dataToWrite +=
               input.backupFileType === 'json'
                 ? JSON.stringify(hits)
-                : JSON.stringify(convertToCsv(hits));
-            await sourceFileApi.saveFile(filePath, dataToWrite, true);
+                : convertToCsv(csvHeaders, hits);
+            await sourceFileApi.saveFile(filePath, dataToWrite, appendFile);
+            dataToWrite = '';
+            appendFile = true;
           }
         }
         return filePath;
@@ -108,28 +114,29 @@ export const useBackupRestoreStore = defineStore('backupRestoreStore', {
   },
 });
 
-const flattenObject = (obj: any, parent: string = '', res: any = {}) => {
-  for (let key in obj) {
-    const propName = parent ? `${parent}.${key}` : key;
-    if (typeof obj[key] === 'object' && obj[key] !== null) {
-      flattenObject(obj[key], propName, res);
-    } else {
-      res[propName] = obj[key];
-    }
-  }
-  return res;
+const buildCsvHeaders = ({
+  mappings,
+}: {
+  mappings: {
+    properties: {
+      [key: string]: unknown;
+    };
+  };
+}) => {
+  return Object.keys(mappings.properties);
 };
 
-const convertToCsv = (data: any[]) => {
-  if (data.length === 0) {
-    return { headers: [], data: [] };
-  }
-
-  const flattenedData = data.map(row => flattenObject(row));
-  const headers = Array.from(new Set(flattenedData.flatMap(row => Object.keys(row))));
-  const csvRows = flattenedData.map(row =>
-    headers.map(header => JSON.stringify(row[header] ?? '')).join(','),
-  );
-
-  return { headers, data: csvRows };
+const convertToCsv = (headers: Array<string>, data: unknown[]) => {
+  return data
+    .map(item =>
+      headers
+        .map(header => {
+          const data = get(item, `_source.${header}`, null);
+          return data === null || !['object', 'symbol', 'function'].includes(typeof data)
+            ? data
+            : JSON.stringify(data);
+        })
+        .join(','),
+    )
+    .join('\r\n');
 };
diff --git a/src/views/backup-restore/components/backup.vue b/src/views/backup-restore/components/backup.vue
@@ -305,7 +305,14 @@ const submitBackup = async () => {
       positiveText: lang.t('dialogOps.confirm'),
       negativeText: lang.t('dialogOps.cancel'),
       onPositiveClick: async () => {
-        await saveBackup(backupInput);
+        saveBackup(backupInput).catch(err => {
+          const error = err as CustomError;
+          message.error(`status: ${error.status}, details: ${error.details}`, {
+            closable: true,
+            keepAliveOnHover: true,
+            duration: 3600,
+          });
+        });
       },
       onNegativeClick: () => {},
     });

diff --git a/tests/fixtures/index.ts b/tests/fixtures/index.ts