diff --git a/eda/eda_api/lib/app.ts b/eda/eda_api/lib/app.ts index 6037bc0bf..a09ffd3ae 100644 --- a/eda/eda_api/lib/app.ts +++ b/eda/eda_api/lib/app.ts @@ -9,6 +9,8 @@ import errorMiddleware from './middleware/error.middleware'; import Router from './router'; const path = require('path'); +/* SDA CUSTOM*/ // Show current node_modules folder +/* SDA CUSTOM*/ console.log("NODE MODULES PATH",require.resolve('lodash')); const database = require('../config/database.config'); const mongoose = require('mongoose'); const compression = require('compression'); diff --git a/eda/eda_api/lib/module/updateModel/service/cleanModel.ts b/eda/eda_api/lib/module/updateModel/service/cleanModel.ts index 0c71d6e44..0754c13c1 100644 --- a/eda/eda_api/lib/module/updateModel/service/cleanModel.ts +++ b/eda/eda_api/lib/module/updateModel/service/cleanModel.ts @@ -1,195 +1,92 @@ +/** +* Este módulo se encarga de limpiar y consolidar los permisos del modelo de datos. +* Maneja la deduplicación de roles, fusión de permisos y sincronización con MongoDB. +*/ + import _ from "lodash"; -import Group, { IGroup } from '../../admin/groups/model/group.model' -import DataSourceSchema from '../../datasource/model/datasource.model' +import Group, { IGroup } from '../../admin/groups/model/group.model'; +import DataSourceSchema from '../../datasource/model/datasource.model'; export class CleanModel { - - public async cleanModel(main_model : any) : Promise { - - let roles = _.cloneDeep(main_model.ds.metadata.model_granted_roles); - - let model = { - "users": [], - "usersName": [], - "none": "", - "table": "", - "column": "", - "global": "", - "permission": "", - "type": "", - "value":[] - } - - let modelAc = { - "users": [], - "usersName": [], - "none": "", - "table": "", - "column": "", - "global": "", - "permission": "", - "type": "", - "value":[] - } - - let groupModel = { - "groups": [], - "groupsName": [], - "none": "", - "table": "", - "column": "", - "global": "", - "permission": "", - "type": "", - "value":[] - } - - let model_granted_roles = [] ; - - for (let i=0;i r.table == roles[i].table - && r.column == roles[i].column - && r.type == roles[i].type - && r.global == roles[i].global - && r.none == roles[i].none - && r.permission == roles[i].permission - && r?.dynamic == roles[i]?.dynamic - ); - if( _.isEmpty(match) == false ){ - if(roles[i].value && match.value ){ - roles[i].value.forEach((e,i)=> { - if( e != match.value[i]){ - match = false; - } - }); - } - } - - - if (_.isEmpty(match) == false && roles[i].type == "users") { - if (!match.users.includes(roles[i].users[0])) {match.users.push(roles[i].users[0]) } ; - if (!match.usersName.includes(roles[i].usersName[0])) {match.usersName.push(roles[i].usersName[0]) } ; - modelAc = match; - - } else if (_.isEmpty(match) == false && roles[i].type == "groups") { - if (!match.groups.includes(roles[i].groups[0])) {match.groups.push(roles[i].groups[0]) } ; - if (!match.groupsName.includes(roles[i].groupsName[0])) {match.groupsName.push(roles[i].groupsName[0]) } ; - groupModel = match; - } - else { - if (_.isEmpty(modelAc.table) == false) {model_granted_roles.push(modelAc) }; - if (_.isEmpty(groupModel.table) == false ) {model_granted_roles.push(groupModel)}; - if (roles[i].type == "groups") { - groupModel = roles[i]; - if (_.isEmpty(groupModel.table) == false) {model_granted_roles.push(groupModel)} ; - } else { - model = roles[i]; - if (_.isEmpty(model.table) == false) {model_granted_roles.push(model)} ; - } - } - } - } - - - //recuperamos los model_granted_roles de mongo, donde se han añadido permisos para SCRM_* - const finder = await DataSourceSchema.find({_id: "111111111111111111111111" }) ; - let mgs = []; - const mgsmap = _.cloneDeep(finder.map(e => mgs = e.ds.metadata.model_granted_roles)); - - function objetosIgualesGrupos(objetoA: any, objetoB: any): boolean { - if (objetoA.groups != undefined && objetoB.groups != undefined - ) return ( - objetoA.groups.join(',') === objetoB.groups.join(',') && - objetoA.groupsName.join(',') === objetoB.groupsName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type - ); - } - - - function objetosIgualesUsuarios(objetoA: any, objetoB: any): boolean { - if (objetoA.users != undefined && objetoB.users != undefined && objetoA?.dynamic && objetoB?.dynamic && objetoA?.value && objetoB?.value ){ - return ( - objetoA.users.join(',') === objetoB.users.join(',') && - objetoA.usersName.join(',') === objetoB.usersName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type && - objetoA?.dynamic === objetoB?.dynamic && - objetoA?.value[0] === objetoB?.value[0] - ); - }else if (objetoA.users != undefined && objetoB.users != undefined && objetoA?.value && objetoB?.value ){ - return ( - objetoA.users.join(',') === objetoB.users.join(',') && - objetoA.usersName.join(',') === objetoB.usersName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type && - objetoA?.value[0] === objetoB?.value[0] - ); - }else if (objetoA.users != undefined && objetoB.users != undefined ){ - return ( - objetoA.users.join(',') === objetoB.users.join(',') && - objetoA.usersName.join(',') === objetoB.usersName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type - ); - } - } - - - // Filtrar objetos únicos grupos - const objetosUnicosGrupos = model_granted_roles.filter((objeto, index, self) => - self.findIndex(other => objetosIgualesGrupos(objeto, other)) === index - ); - - - - // Filtrar objetos únicos usuarios - const objetosUnicosUsuarios = model_granted_roles.filter((objeto, index, self) => - self.findIndex(other => objetosIgualesUsuarios(objeto, other)) === index - ); - - model_granted_roles = objetosUnicosGrupos.concat(objetosUnicosUsuarios); - - model_granted_roles.forEach( r=> { - r.source = 'update_model'; - } - ); - - // Recuperando los permisos provenientes de SinergiaCRM - // la propiedad source --> "EDA" indica que el permiso proviene de la applicacion y no de la base de datos - if(mgsmap.length!==0) { - const userRoles = mgsmap[0].filter( (r:any) => { - return r?.source === 'SDA' && !r.groupsName.find( e => e.startsWith('SCRM_')) - }); - - // Agregando los permisos agregados previamente en la aplicacion. - const all_roles = [ ...model_granted_roles, ...userRoles]; - main_model.ds.metadata.model_granted_roles = all_roles; - } - - return main_model; - } - } - + /** + * Limpia y consolida los permisos del modelo + * @param main_model Modelo de datos a procesar + * @returns Modelo procesado con permisos consolidados + */ + public async cleanModel(main_model: any): Promise { + // Clonar roles para no modificar original + const roles = _.cloneDeep(main_model.ds.metadata.model_granted_roles); + const model_granted_roles: any[] = []; + const mapRoles = new Map(); + + /** + * Añade o actualiza un rol en el mapa de roles + * @param role Rol a procesar + * @param key Clave única del rol + */ + const addOrUpdateRole = (role: any, key: string) => { + const existingRole = mapRoles.get(key); + if (existingRole) { + // Fusionar usuarios o grupos según tipo + if (role.type === "users") { + existingRole.users = Array.from(new Set([...existingRole.users, ...role.users])); + existingRole.usersName = Array.from(new Set([...existingRole.usersName, ...role.usersName])); + } else if (role.type === "groups") { + existingRole.groups = Array.from(new Set([...existingRole.groups, ...role.groups])); + existingRole.groupsName = Array.from(new Set([...existingRole.groupsName, ...role.groupsName])); + } + } else { + // Crear nuevo rol si no existe + mapRoles.set(key, _.cloneDeep(role)); + } + }; + + // Procesar cada rol y generar clave única + roles.forEach((role: any) => { + const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.dynamic}-${role.value?.join(',')}`; + addOrUpdateRole(role, key); + }); + + // Convertir mapa a array + mapRoles.forEach((value) => model_granted_roles.push(value)); + + // Obtener permisos existentes de MongoDB + const finder = await DataSourceSchema.find({ _id: "111111111111111111111111" }); + const mgsmap = finder + .map((e) => e.ds.metadata.model_granted_roles) + .reduce((acc, val) => acc.concat(val), []); + + /** + * Filtra roles duplicados usando una clave compuesta + * @param roles Array de roles a filtrar + * @param comparator Función de comparación + * @returns Array de roles únicos + */ + const filterUniqueRoles = (roles: any[], comparator: (a: any, b: any) => boolean) => { + const seen = new Map(); + return roles.filter((role) => { + const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.users?.join(',')}-${role.groups?.join(',')}`; + if (seen.has(key)) return false; + seen.set(key, role); + return true; + }); + }; + + // Obtener roles únicos y marcar origen + const uniqueRoles = filterUniqueRoles(model_granted_roles, _.isEqual); + uniqueRoles.forEach((role) => (role.source = "update_model")); + + // Combinar roles según existencia de permisos en MongoDB + if (mgsmap.length) { + // Filtrar roles de usuario SDA no-SCRM + const userRoles = mgsmap.filter( + (r: any) => r?.source === "SDA" && !r.groupsName.some((name: string) => name.startsWith("SCRM_")) + ); + main_model.ds.metadata.model_granted_roles = [...uniqueRoles, ...userRoles]; + } else { + main_model.ds.metadata.model_granted_roles = uniqueRoles; + } + + return main_model; + } +} \ No newline at end of file diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index f965e412e..38fd38583 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -1,266 +1,472 @@ /** - * User and Group Synchronization Service - * Part of the updateModel module - * - * This service handles the synchronization of users and groups between SinergiaCRM - * and SinergiaDA (Sinergia Data Analytics) systems. It is a crucial component of - * the updateModel process, ensuring that user and group data is consistently - * maintained across both platforms. Key functions include: - * - * 1. Importing users from SinergiaCRM to SinergiaDA, creating or updating as needed. - * 2. Importing and creating groups from SinergiaCRM roles in SinergiaDA. - * 3. Synchronizing user-group relationships based on CRM roles and SDA-specific rules. - * 4. Handling special cases for admin users and SinergiaCRM-specific groups (prefixed with 'SCRM_'). - * 5. Removing inactive users and deleted groups to maintain system consistency. - * 6. Updating the MongoDB database in SinergiaDA with the synchronized data. - * - * The userAndGroupsToMongo class contains two primary methods: - * - crm_to_eda_UsersAndGroups: Initiates the synchronization process. - * - syncronizeUsersGroups: Performs detailed synchronization logic. - * - * This service is a critical part of maintaining data integrity between SinergiaCRM - * and SinergiaDA, ensuring that analytics and reporting in SinergiaDA accurately - * reflect the current state of users and groups in the CRM system. - * - * Note: This service assumes the existence of User and Group models in the SinergiaDA - * MongoDB database and relies on user and role data structures from SinergiaCRM. +* Módulo de sincronización de usuarios y permisos entre CRM y MongoDB. +* +* Este módulo implementa la lógica para mantener sincronizados los usuarios y grupos entre +* un CRM basado en MariaDB y una base de datos MongoDB. Las principales funcionalidades son: +* +* - Sincronización de usuarios: importa usuarios del CRM, mantiene contraseñas actualizadas +* y elimina usuarios inactivos preservando cuentas del sistema. +* +* - Gestión de grupos: mantiene grupos SCRM_ sincronizados con el CRM mientras preserva +* grupos propios de SDA (EDA_ADMIN, EDA_RO, etc). +* +* - Relaciones bidireccionales: garantiza consistencia en las relaciones usuario-grupo +* tanto desde la perspectiva del usuario como del grupo. +* +* - Caché optimizada: implementa un sistema de caché que se reconstruye en cada ejecución +* para optimizar consultas a MongoDB durante el proceso. +* +* - Procesamiento por lotes: realiza operaciones de escritura en lotes para mejor rendimiento. +* +* - Tratamiento especial para grupo EDA_ADMIN: mantiene sincronizada la pertenencia a este grupo +* según sda_def_user_groups. +*/ +import mongoose, { connections, Model, mongo, Mongoose, QueryOptions } from "mongoose"; +import User, { IUser } from "../../admin/users/model/user.model"; +import Group, { IGroup } from "../../admin/groups/model/group.model"; +import { Console } from "console"; + +// Desactiva warnings obsoletos de mongoose +mongoose.set("useFindAndModify", false); + +// Define estructura de usuario en MongoDB +interface MongoUser extends mongoose.Document { + email: string; + password: string; + name: string; + role: mongoose.Types.ObjectId[]; + active?: number; +} + +// Define estructura de grupo en MongoDB +interface MongoGroup extends mongoose.Document { + _id: mongoose.Types.ObjectId; + name: string; + role: string; + users: mongoose.Types.ObjectId[]; +} + +// Define estructura de usuario en CRM +interface CRMUser { + name: string; + email: string; + password: string; + active: number; +} + +// Define estructura de rol en CRM +interface CRMRole { + name: string; + user_name?: string; +} + +// Define estructura de resultados de sincronización +interface SyncResults { + inserted: number; + updated: number; + deleted: number; +} + +/** + * Implementa un sistema de caché para usuarios y grupos en MongoDB + * Se reconstruye en cada ejecución para mantener datos actualizados */ +class DataCache { + private static instance: DataCache; + private cache: Map; + + private constructor() { + this.cache = new Map(); + } + + /** + * Obtiene o crea la instancia única del caché + * @returns Instancia del caché limpia + */ + static getInstance(): DataCache { + if (!DataCache.instance) { + DataCache.instance = new DataCache(); + } + DataCache.instance.clearCache(); + return DataCache.instance; + } -import mongoose, { - connections, - Model, - mongo, - Mongoose, - QueryOptions -} from 'mongoose' -import User, { IUser } from '../../admin/users/model/user.model' -import Group, { IGroup } from '../../admin/groups/model/group.model' + /** + * Elimina todos los datos almacenados en caché + */ + private clearCache(): void { + this.cache.clear(); + } -mongoose.set('useFindAndModify', false); + /** + * Recupera usuarios de MongoDB usando caché + * @returns Lista de usuarios + */ + async getUsers(): Promise { + const cacheKey = 'all_users'; + if (!this.cache.has(cacheKey)) { + const users = await User.find().lean() as unknown as MongoUser[]; + this.cache.set(cacheKey, users); + } + return this.cache.get(cacheKey) as MongoUser[]; + } + + /** + * Recupera grupos de MongoDB usando caché + * @returns Lista de grupos + */ + async getGroups(): Promise { + const cacheKey = 'all_groups'; + if (!this.cache.has(cacheKey)) { + const groups = await Group.find().lean() as unknown as MongoGroup[]; + this.cache.set(cacheKey, groups); + } + return this.cache.get(cacheKey) as MongoGroup[]; + } +} export class userAndGroupsToMongo { - static async crm_to_eda_UsersAndGroups(users: any, roles: any) { - // Sync users and groups from CRM to EDA - let mongoUsers = await User.find() - - // Initialize users - for (let i = 0; i < users.length; i++) { - let existe = mongoUsers.find(e => e.email == users[i].email) - if (!existe) { - let user = new User({ - name: users[i].name, - email: users[i].email, - password: users[i].password, - role: [] - }) + /** + * Coordina la sincronización completa de usuarios y grupos entre CRM y MongoDB + * Gestiona la creación de índices y ejecuta las sincronizaciones en paralelo + * + * @param users Lista de usuarios del CRM a sincronizar + * @param roles Lista de roles/grupos del CRM a sincronizar + * @returns Resultados de la sincronización + */ + static async crm_to_eda_UsersAndGroups(users: CRMUser[], roles: CRMRole[]) { + console.time("Total usersAndGroupsToMongo"); + console.log(`Starting sync: ${users.length} users and ${roles.length} role assignments`); + try { - await user.save() - } catch (err) { - console.log( - 'usuario ' + - user.name + - ' (Could not insert into the MongoDB database.)' ) + // Asegura índices únicos + await User.collection.createIndex({ email: 1 }, { unique: true }); + await Group.collection.createIndex({ name: 1 }, { unique: true }); + + const userSyncResults = await this.optimizedUserSync(users); + const groupSyncResults = await this.optimizedGroupSync(roles); + + console.log('Sync completed:', { + users: userSyncResults, + groups: groupSyncResults + }); + + console.timeEnd("Total usersAndGroupsToMongo"); + return { userSyncResults, groupSyncResults }; + } catch (error) { + console.error("Sync error:", error); + throw error; } - } else { - await User.findOneAndUpdate({ name: users[i].name }, { password: users[i].password }); - } } - // Initialize groups - let mongoGroups = await Group.find() - mongoUsers = await User.find() - const unique_groups = [...new Set(roles.map(item => item.name))] - - for (let i = 0; i < unique_groups.length; i++) { - let existe = mongoGroups.find(e => e.name == unique_groups[i]) - if ( - !existe && - unique_groups[i] != 'EDA_ADMIN' && - unique_groups[i] != 'EDA_RO' && - unique_groups[i] != 'EDA_DATASOURCE_CREATOR' - ) { - let group = new Group({ - role: 'EDA_USER_ROLE', - name: unique_groups[i], - users: [] - }) + /** + * Gestiona la sincronización optimizada de usuarios + * Procesa actualizaciones e inserciones en lotes + * + * @param crmUsers Lista de usuarios del CRM + * @returns Estadísticas de la sincronización + */ + private static async optimizedUserSync(crmUsers: CRMUser[]): Promise { + console.time('userSync'); + try { - await group.save() - console.log(`Group ${group.name} inserted successfully`) - } catch (err) { - console.log(`Group ${group.name} already exists, skipped`) + const mongoUsers = await DataCache.getInstance().getUsers(); + const emailToMongoUser = new Map( + mongoUsers.map(user => [user.email, user]) + ); + + const [syncResult, deleteResult] = await Promise.all([ + this.synchronizeUsers(crmUsers, emailToMongoUser), + this.removeInactiveUsers(crmUsers) + ]); + + console.timeEnd('userSync'); + + return { + inserted: syncResult?.insertedCount || 0, + updated: syncResult?.modifiedCount || 0, + deleted: deleteResult?.deletedCount || 0 + }; + } catch (error) { + console.error('User sync error:', error); + throw error; } - } } - // Synchronize users and groups - await this.syncronizeUsersGroups(mongoUsers, mongoGroups, users, roles); - } - - static async syncronizeUsersGroups( - mongoUsers: any, - mongoGroups: any, - crmUsers: any, - crmRoles: any - ) { - // Remove inactive users from CRM - mongoUsers.forEach(a => { - let existe = crmUsers.find(u => u.email === a.email); - if (existe) { - if ( - a.email !== 'eda@sinergiada.org' && - a.email !== 'eda@jortilles.com' && - a.email !== 'edaanonim@jortilles.com' && - existe.active == 0) { - User.deleteOne({ email: a.email }) - .then(function () { - }) - .catch(function (error) { - console.log('Error deleting user:', a.email, 'Details:', error); - }) + /** + * Procesa usuarios en lotes para sincronización + * Maneja creación de nuevos usuarios y actualización de contraseñas + * + * @param crmUsers Lista de usuarios a sincronizar + * @param emailToMongoUser Mapa de usuarios existentes + * @returns Resultado de operaciones bulk + */ + private static async synchronizeUsers(crmUsersRaw: CRMUser[], emailToMongoUser: Map) { + const batchSize = 100; + const operations: any[] = []; + + let crmUsers = crmUsersRaw.filter(element => element.active === 1); + + for (let i = 0; i < crmUsers.length; i += batchSize) { + const batch = crmUsers.slice(i, i + batchSize); + + batch.forEach(crmUser => { + const existingUser = emailToMongoUser.get(crmUser.email); + + if (!existingUser) { + operations.push({ + insertOne: { + document: { + name: crmUser.name, + email: crmUser.email, + password: crmUser.password, + role: [] + } + } + }); + } else if (existingUser.password !== crmUser.password) { + operations.push({ + updateOne: { + filter: { email: crmUser.email }, + update: { $set: { password: crmUser.password } } + } + }); + } + }); } - } - }) - - // Remove deleted CRM groups - mongoGroups.forEach(a => { - if( a.name.startsWith('SCRM_') ){ - let existe = crmRoles.find(u => u.name === a.name); - if(!existe){ - Group.deleteOne( {name: a.name} ).then( function(){ console.log( a.name + ' deleted')}) + + if (operations.length > 0) { + return await User.collection.bulkWrite(operations, { ordered: false }); } - } - }); - - // Helper functions to check user existence in CRM - const userExistsInCRM = (user, crmUsers) => { - return crmUsers.some(crmUser => crmUser.email === user.email && crmUser.active == 1); - }; - - const userExistedInCRM = (user, crmUsers) => { - return crmUsers.some(crmUser => crmUser.email === user.email); - }; - - // Synchronize groups and users - await mongoGroups.forEach(async (group) => { - if (group.name.startsWith('SCRM_')) { - // For SCRM_ groups, maintain SDA users and sync with CRM - const crmUsersInGroup = crmRoles - .filter(role => role.name === group.name) - .map(role => mongoUsers.find(u => u.email === role.user_name)) - .filter(user => user) - .map(user => user._id); - group.users = [ - ...group.users.filter(userId => { - const user = mongoUsers.find(u => u._id.toString() === userId.toString()); - return user && !userExistsInCRM( user, crmUsers); - }), - ...crmUsersInGroup + return null; + } + + + + /** + * Elimina usuarios inactivos del CRM preservando usuarios del sistema + * + * @param crmUsers Lista de usuarios del CRM + * @returns Resultado de la operación de eliminación + */ + private static async removeInactiveUsers(crmUsers: CRMUser[]) { + const excludedEmails = [ + 'eda@sinergiada.org', + 'eda@jortilles.com', + 'edaanonim@jortilles.com' ]; - // Add new CRM users to the group - const newCrmUsersInGroup = crmRoles - .filter(role => role.name === group.name) - .map(role => mongoUsers.find(u => u.email === role.user_name && userExistsInCRM(u, crmUsers) )) - .filter(user => user && !group.users.includes(user._id)) - .map(user => user._id); - - group.users = [...group.users, ...newCrmUsersInGroup]; - - } else { - // For non-SCRM_ groups, maintain SDA users and update CRM users - group.users = group.users.filter(userId => { - const user = mongoUsers.find(u => u._id.toString() === userId.toString()); - if (!user) return false; - if (userExistedInCRM(user, crmUsers)) { - return userExistsInCRM(user, crmUsers) ; - } else { - return true; - } - }); - } - }); - - // Update user roles - await mongoUsers.forEach(async (user) => { - if( userExistsInCRM( user, crmUsers)) { - // Update CRM users, maintain non-SCRM_ roles - const nonCRMRoles = user.role.filter(roleId => { - const group = mongoGroups.find(g => g._id.toString() === roleId.toString() && !g.name.startsWith('SCRM_') ); - return group; + const crmEmails = new Set(crmUsers.map(user => user.email)); + const activeEmails = new Set( + crmUsers + .filter(user => user.active === 1) + .map(user => user.email) + ); + + const bulkDelete = { + deleteMany: { + filter: { + $and: [ + { email: { $in: Array.from(crmEmails) } }, + { email: { $nin: [...excludedEmails, ...Array.from(activeEmails)] } } + ] + } + } + }; + + return await User.collection.bulkWrite([bulkDelete]); + } + + /** + * Gestiona la sincronización optimizada de grupos + * Coordina creación de grupos y actualización de membresías + * + * @param roles Lista de roles del CRM + * @returns Estadísticas de sincronización + */ + private static async optimizedGroupSync(roles: CRMRole[]) { + console.time('groupSync'); + + try { + const mongoGroups = await DataCache.getInstance().getGroups(); + const nameToMongoGroup = new Map( + mongoGroups.map(group => [group.name, group]) + ); + + const [syncResult, userAssignments] = await Promise.all([ + this.synchronizeGroups(roles, nameToMongoGroup), + this.updateGroupUsers(roles, mongoGroups) + ]); + + console.timeEnd('groupSync'); + + return { + inserted: syncResult?.insertedCount || 0, + updated: syncResult?.modifiedCount || 0, + userAssignments + }; + } catch (error) { + console.error('Group sync error:', error); + throw error; + } + } + + /** + * Crea nuevos grupos del CRM en MongoDB + * Excluye grupos especiales del sistema + * + * @param roles Lista de roles del CRM + * @param nameToMongoGroup Mapa de grupos existentes + * @returns Resultado de operaciones bulk + */ + private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map) { + const uniqueGroups = [...new Set(roles.map(item => item.name))]; + const operations: any[] = []; + + uniqueGroups.forEach(groupName => { + if (!nameToMongoGroup.has(groupName) && + !['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(groupName)) { + + operations.push({ + insertOne: { + document: { + role: 'EDA_USER_ROLE', + name: groupName, + users: [] + } + } + }); + } }); - // Add CRM roles - const crmRolesForUser = crmRoles - .filter(role => role.user_name === user.email) - .map(role => mongoGroups.find(g => g.name === role.name)) - .filter(group => group) - .map(group => group._id); + if (operations.length > 0) { + return await Group.collection.bulkWrite(operations, { ordered: false }); + } + + return null; + } + + /** + * Actualiza las membresías de usuarios en grupos + * Mantiene grupos SCRM_ sincronizados con CRM y preserva otros grupos + * + * @param roles Lista de roles del CRM + * @param mongoGroups Lista de grupos en MongoDB + * @returns Resultado de operaciones bulk + */ + private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[]) { + const groupUpdates = new Map>(); + const userUpdates = new Map>(); + + const usersCache = await DataCache.getInstance().getUsers(); + const emailToId = new Map(usersCache.map(u => [u.email, u._id])); + const idToEmail = new Map(usersCache.map(u => [u._id.toString(), u.email])); + + // Mapeo de asignaciones CRM + const crmUsersGroups = new Map>(); + roles.forEach(role => { + if (role.user_name && role.name.startsWith('SCRM_')) { + if (!crmUsersGroups.has(role.user_name)) { + crmUsersGroups.set(role.user_name, new Set()); + } + crmUsersGroups.get(role.user_name)?.add(role.name); + } + }); - user.role = [...new Set([...nonCRMRoles, ...crmRolesForUser])]; - } - }); - - // Add admin user to EDA_ADMIN group - await mongoGroups.find(i => i.name === 'EDA_ADMIN').users.push('135792467811111111111111') - let user = await mongoUsers.find(i => i.email === ('eda@jortilles.com') ) ; - if(user){ - user.role.push('135792467811111111111110'); - }else{ - user = await mongoUsers.find(i => i.email === ('eda@sinergiada.org' ) ) - if(user){ - user.role.push('135792467811111111111110'); - }else{ - console.log('Error: Failed to assign admin role to user'); - } + // Obtener usuarios que deben estar en EDA_ADMIN desde sda_def_user_groups + const edaAdminUsers = new Set( + roles.filter(role => + role.user_name && + role.name === 'EDA_ADMIN' + ).map(role => role.user_name!) + ); + + // Procesamiento de grupos + mongoGroups.forEach(group => { + const uniqueUsers = new Set(); + + if (group.name === 'EDA_ADMIN') { + // Mantener usuarios existentes + group.users.forEach(userId => { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + + // Añadir usuarios de sda_def_user_groups + usersCache.forEach(user => { + if (edaAdminUsers.has(user.email)) { + uniqueUsers.add(user._id); + this.updateUserRoles(userUpdates, user._id.toString(), group._id); + } + }); + } else if (group.name.startsWith('SCRM_') || group.name === 'EDA_ADMIN') { + group.users.forEach(userId => { + const userEmail = idToEmail.get(userId.toString()); + if (userEmail && crmUsersGroups.get(userEmail)?.has(group.name)) { + uniqueUsers.add(userId); + } + }); + + usersCache.forEach(user => { + if (crmUsersGroups.get(user.email)?.has(group.name)) { + uniqueUsers.add(user._id); + } + }); + + uniqueUsers.forEach(userId => { + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + } else { + group.users.forEach(userId => { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + } + + if (uniqueUsers.size > 0) { + groupUpdates.set(group.name, uniqueUsers); + } + }); + + // Preparación de operaciones bulk + const groupOperations = Array.from(groupUpdates.entries()).map(([groupName, userIds]) => ({ + updateOne: { + filter: { name: groupName }, + update: { $set: { users: Array.from(userIds) } } + } + })); + + const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => ({ + updateOne: { + filter: { _id: new mongoose.Types.ObjectId(userId) }, + update: { $set: { role: Array.from(groupIds) } } + } + })); + + return Promise.all([ + groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, + userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null + ]); } - // Save changes to database - await mongoGroups.forEach(async r => { - try { - await Group.updateOne({ name: r.name }, { $unset: { users: {} } }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - } catch (err) { - console.log(err); - } - try { - await Group.updateOne({ name: r.name }, { $addToSet: { users: r.users } }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - } catch (err) { - console.log(err); - } - }) - - const newGroupsInMongo = await Group.find(); - const newGroupsIDInMongo = newGroupsInMongo.map(g=>g._id.toString()); - - // Update user roles - await mongoUsers.forEach(async user => { - user.role = user.role.filter( r => newGroupsIDInMongo.includes(r.toString() ) ) - try { - await User.updateOne({ email: user.email }, { $unset : {role: {}} }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - - await User.updateOne({ email: user.email }, { $addToSet : {role: user.role} }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - } catch (err) {} - }) - } + /** + * Actualiza las relaciones de roles de usuario en el mapa de actualizaciones + * @param userUpdates - Mapa de actualizaciones usuario-roles + * @param userId - ID del usuario + * @param groupId - ID del grupo a añadir + */ +private static updateUserRoles( + userUpdates: Map>, + userId: string, + groupId: mongoose.Types.ObjectId +): void { + // Inicializar set si no existe + if (!userUpdates.has(userId)) { + userUpdates.set(userId, new Set()); + } + // Añadir nuevo groupId al set de roles + userUpdates.get(userId)?.add(groupId); } + +} \ No newline at end of file diff --git a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts index c355cd72d..71c584f8b 100644 --- a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts +++ b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts @@ -1,3 +1,10 @@ +/** +* Este módulo gestiona la actualización del modelo de datos de SinergiaDA. +* Maneja la sincronización entre la base de datos MariaDB del CRM y MongoDB, +* incluyendo la estructura de tablas, columnas, relaciones, enumeraciones y +* permisos de usuarios/grupos. +*/ + import { kMaxLength } from "buffer"; import { NextFunction, Request, Response } from "express"; import Group, { IGroup } from "../admin/groups/model/group.model"; @@ -6,7 +13,7 @@ import { EnCrypterService } from "../../services/encrypter/encrypter.service"; import { userAndGroupsToMongo } from "./service/usersAndGroupsToMongo"; import { Enumerations } from "./service/enumerations"; import { pushModelToMongo } from "./service/push.Model.to.Mongo"; - +import path from 'path'; import fs from "fs"; import { CleanModel } from "./service/cleanModel"; @@ -14,582 +21,700 @@ const mariadb = require("mariadb"); const sinergiaDatabase = require("../../../config/sinergiacrm.config"); export class updateModel { - /** Updates the SinergiaDA data model of an instance */ - static async update(req: Request, res: Response) { - let crm_to_eda: any = {}; - let modelToExport: any = {}; - let grantedRolesAt: any = []; - let enumerator: any; - let connection: any; - console.time("UpdateModel"); - connection = await mariadb.createConnection(sinergiaDatabase.sinergiaConn); - console.timeLog("UpdateModel", "(Create connection)"); - - try { - /** Checks if columns and tables defined in the model exist */ - await updateModel.checkSinergiaModel(connection); - console.timeLog("UpdateModel", "(Checking model)"); - - // Connect to client database to extract data and create our EDA model - // Select tables - await connection - .query( - " select `table`, label , description, visible from sda_def_tables sdt union all " + - " select distinct master_table , master_table ,master_table , 0 as visible " + - " from sda_def_enumerations sde union all " + - " select distinct bridge_table , bridge_table ,bridge_table , 0 as visible " + - " from sda_def_enumerations sde " + - " where bridge_table != '' " - ) - .then(async (rows, err1) => { - if (err1) { - console.log("Error getting tables list"); - throw err1; - } - let tables = rows; - // Select columns - const my_query = - " select sdc.`table`, sdc.`column`,`type`,sdc.label, sdc.description, sdc.decimals, sdc.aggregations, sdc.visible, sdc.stic_type, sdc.sda_hidden " + - " FROM sda_def_columns sdc " + - " union " + - " select master_table , master_id , 'text', master_id , master_id , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " union " + - " select master_table , master_column , 'text', master_column , master_column , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " union " + - " select bridge_table , source_bridge , 'text', source_bridge , source_bridge , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " where bridge_table != '' " + - " union " + - " select bridge_table , target_bridge , 'text', target_bridge , target_bridge , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " where bridge_table != '' "; - await connection.query(my_query).then(async rows => { - let columns = rows; - // Select relationships - await connection - .query( - ` - SELECT distinct source_table, source_column, target_table, target_column, label , 0 as direccion - FROM sda_def_relationships - union - SELECT target_table as source_table, target_column as source_column , - source_table as target_table , source_column as target_column, label as label , 1 as direccion - FROM sda_def_relationships - where source_table != target_table - union - SELECT source_table , source_column , - master_table , master_id as target_column, 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table is null or bridge_table = '' - union - SELECT master_table , master_id , - source_table , source_column , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table is null or bridge_table = '' - union - SELECT source_table , source_bridge as source_column , - bridge_table , source_bridge, 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' - union - SELECT bridge_table , source_bridge, - source_table , source_bridge as source_column , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' - union - SELECT bridge_table , target_bridge, - master_table , master_id , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' - union - SELECT master_table , master_id, - bridge_table , target_bridge , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' ` - ) - .then(async rows => { - let relations = rows; - // Select users - await connection - .query( - "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE password IS NOT NULL ;" - ) - .then(async users => { - let users_crm = users; - // Select EDA roles - await connection - .query( - 'select "EDA_USER_ROLE" as role, b.name, "" as user_name from sda_def_groups b union select "EDA_USER_ROLE" as role, g.name as name , g.user_name from sda_def_user_groups g; ' - ) - .then(async role => { - let roles = role; - await connection - .query( - ' select distinct a.user_name as name, a.`table`, "id" as `column`, a.`group` from sda_def_permissions a where a.`group` != "" ; ' - ) - .then(async granted => { - let fullTablePermissionsForRoles = granted; - // Select enumerations - await connection - .query( - " select source_table , source_column , master_table, master_id, master_column, bridge_table, source_bridge, target_bridge, stic_type, info from sda_def_enumerations sde ;" - ) - .then(async enums => { - let ennumeration = enums; - await connection - .query(" select user_name as name, `table` from sda_def_permissions ") - .then(async permi => { - let fullTablePermissionsForUsers = permi; - - await connection - .query( - " select distinct user_name as name, `table`, 'id' as `column`, group_concat( distinct `group`) as `group` from sda_def_permissions where `group` != '' group by 1,2,3 " - ) - .then(async permiCol => { - let dynamicPermisssionsForGroup = permiCol; - /** Now that we have all data, proceed to build the model */ - const query = - 'select user_name as name, `table` as tabla , `column` as columna from sda_def_permissions where stic_permission_source in ( "ACL_ALLOW_OWNER")'; - await connection.query(query).then(async customUserPermissionsValue => { - console.timeLog("UpdateModel", "(Run MariaDB queries)"); - let dynamicPermisssionsForUser = customUserPermissionsValue; - - try { - crm_to_eda = await userAndGroupsToMongo.crm_to_eda_UsersAndGroups( - users_crm, - roles - ); - console.timeLog("UpdateModel", "(Syncs users and groups)"); - } catch (e) { - console.log("Error 1", e); - res.status(500).json({ status: "ko" }); - } - try { - grantedRolesAt = await updateModel.grantedRolesToModel( - fullTablePermissionsForRoles, - tables, - fullTablePermissionsForUsers, - dynamicPermisssionsForGroup, - dynamicPermisssionsForUser - ); - console.timeLog("UpdateModel", "(Converts CRM roles to EDA)"); - } catch (e) { - console.log("Error 2", e); - res.status(500).json({ status: "ko" }); - } - - try { - modelToExport = updateModel.createModel( - tables, - columns, - relations, - grantedRolesAt, - ennumeration, - res - ); - console.timeLog("UpdateModel", "(Creating Model)"); - } catch (e) { - console.log("Error 3", e); - res.status(500).json({ status: "ko" }); - } - }); - - connection.end(); - }); - }); - }); - }); - }); - }); - }); - }); - }); - } catch (e) { - console.log("Error : ", e); - } - } - /** Checks if columns and tables defined in the model exist */ - static async checkSinergiaModel(con: any) { - let tablas = []; - let columns = []; - let successfulQueries = 0; - - try { - const dataset = await con - .query("select sdc.`table` as tabla, sdc.`column` as `column` FROM sda_def_columns sdc") - .catch(err => { - console.log("Error retrieving tables to check:", err); - throw err; - }); - - tablas = [...new Set(dataset.map(item => item.tabla))]; - - for (const tabla of tablas) { - columns = [...new Set(dataset.map(item => (tabla === item.tabla ? item.column : null)))].filter( - item => item != null - ); - - const sql = " select " + columns.toString() + " from " + tabla + " limit 1 \n"; - let nexSql = sql.replace("select ,", "select ").replace(", from", " from "); - - try { - await con.query(nexSql); - successfulQueries++; - } catch (err) { - console.log(`Error executing query for table ${tabla}:`, err); - } - } - } catch (err) { - console.log("Error in model check process:", err); - } - } - - /** Generates and processes model roles */ - static async grantedRolesToModel( - fullTablePermissionsForRoles: any, - crmTables: any, - fullTablePermissionsForUsers: any, - dynamicPermisssionsForGroup: any, - dynamicPermisssionsForUser: any - ) { - const destGrantedRoles = []; - let gr, - gr2, - gr3, - gr4, - gr5 = {}; - - const usersFound = await User.find(); - const mongoGroups = await Group.find(); - - fullTablePermissionsForRoles.forEach(line => { - let match = mongoGroups.filter(i => { - return i.name === line.group; - }); - let mongoId: String; - let mongoGroup: String; - if (match.length == 1 && line.group !== undefined) { - mongoId = match[0]._id.toString(); - mongoGroup = match[0].name.toString(); - if (line.name != null) { - // Process group converted to user - const found = usersFound.find(i => i.email == line.name); - gr = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "users" - }; - } else { - gr = { - groups: [mongoId], - groupsName: [mongoGroup], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "groups" - }; - } - - destGrantedRoles.push(gr); - } - }); - - fullTablePermissionsForUsers.forEach(line => { - const found = usersFound.find(i => i.email == line.name); - if (found) { - gr3 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "users" - }; - destGrantedRoles.push(gr3); - } - }); - - dynamicPermisssionsForGroup.forEach(line => { - const match = mongoGroups.filter(i => { - return i.name === line.group.split(",")[0]; - }); - let mongoId: String; - if (match.length == 1 && line.group !== undefined) { - mongoId = match[0]._id.toString(); - let group_name: String = " '" + line.group + "' "; - let table_name: String = " '" + line.table + "' "; - let valueAt: String = - " select record_id from sda_def_security_group_records" + - " where `group` in ( " + - group_name.split(",").join("','") + - ") and `table` = " + - table_name; - if (line.name != null) { - // Process group converted to user - const found = usersFound.find(i => i.email == line.name); - gr4 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: line.column, - dynamic: true, - global: false, - type: "users", - value: [valueAt] - }; - let valueAt2: String = " select `id` from " + line.table + " where `assigned_user_name` = 'EDA_USER' "; - gr5 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "id", - global: false, - permission: true, - dynamic: true, - type: "users", - value: [valueAt2] - }; - destGrantedRoles.push(gr4); - destGrantedRoles.push(gr5); - } else { - console.log("Error: Direct group permissions found - not allowed in this context"); - } - - destGrantedRoles.push(gr4); - } - }); - - dynamicPermisssionsForUser.forEach(line => { - const found = usersFound.find(i => i.email == line.name); - if (found) { - let valueAt: String = - "select `" + line.columna + "` from " + line.tabla + " where `" + line.columna + "` = 'EDA_USER' "; - gr5 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.tabla, - column: line.columna, - global: false, - permission: true, - dynamic: true, - type: "users", - value: [valueAt] - }; - destGrantedRoles.push(gr5); - } - }); - - return destGrantedRoles; - } - - static createModel( - tables: any, - columns: any, - relations: any, - grantedRoles: any, - ennumeration: any, - res: any - ): string[] { - let visible = false; - - /** Process and generate tables structure */ - const destTables = []; - for (let i = 0; i < tables.length; i++) { - if (tables[i].visible == 1) { - visible = true; - } else { - visible = false; - } - - var tabla = { - table_name: tables[i].table, - columns: [], - relations: [], - display_name: { - default: tables[i].label, - localized: [] - }, - description: { - default: tables[i].description, - localized: [] - }, - visible: visible, - table_granted_roles: [], - table_type: [], - tableCount: 0, - no_relations: [] - }; - - const destColumns: any[] = updateModel.getColumnsForTable(tables[i].table, columns, ennumeration); - tabla.columns = destColumns; - - const destRelations: any[] = updateModel.getRelations(tables[i].table, relations); - tabla.relations = destRelations; - - destTables.push(tabla); - } - - this.extractJsonModelAndPushToMongo(destTables, grantedRoles, res); - - return destTables; - } - static getAggretations(aggregations: string) { - const aggArray = aggregations.split(","); - const agg = []; - if (aggArray.indexOf("sum") >= 0) { - agg.push({ value: "sum", display_name: "Sum" }); - } - if (aggArray.indexOf("avg") >= 0) { - agg.push({ value: "avg", display_name: "Average" }); - } - if (aggArray.indexOf("max") >= 0) { - agg.push({ value: "max", display_name: "Maximum" }); - } - if (aggArray.indexOf("min") >= 0) { - agg.push({ value: "min", display_name: "Minimum" }); - } - if (aggArray.indexOf("count") >= 0) { - agg.push({ value: "count", display_name: "Count Values" }); - } - if (aggArray.indexOf("count_distinct") > 0) { - agg.push({ value: "count_distinct", display_name: "Distinct Values" }); - } - agg.push({ value: "none", display_name: "None" }); - return agg; - } - - /** Retrieves and processes columns for a specific table */ - static getColumnsForTable(table: string, columns: any, ennumeration: any) { - const destColumns = []; - - const agg_none = [ - { - value: "none", - display_name: "none" - } - ]; - - let agg_used = {}; - let colVisible = false; - - for (let i = 0; i < columns.length; i++) { - let c = columns[i]; - - // Determine aggregators based on column types - if (columns[i].type) { - agg_used = this.getAggretations(columns[i].aggregations); - } else { - agg_used = agg_none; - } - - if (columns[i].visible == 1) { - colVisible = true; - } else { - colVisible = false; - } - - if (c.table == table) { - c = { - column_name: columns[i].column, - column_type: columns[i].type == "enumeration" ? "text" : columns[i].type, - display_name: { - default: columns[i].label, - localized: [] - }, - description: { - default: columns[i].description, - localized: [] - }, - aggregation_type: agg_used, - visible: colVisible, - minimumFractionDigits: columns[i].decimals, - column_granted_roles: [], - row_granted_roles: [], - tableCount: 0, - valueListSource: {}, - hidden: columns[i].sda_hidden - }; - - // Process valueListSource for each column based on source_table - const foundTable = ennumeration.filter(j => j.source_table == table); - foundTable.forEach(u => { - if (u.source_column == c.column_name) { - c.valueListSource = Enumerations.enumeration_to_column(u); + + /** + * Actualiza el modelo de datos de una instancia de SinergiaDA + * Extrae datos del CRM, procesa la estructura y sincroniza con MongoDB + * @param req Petición HTTP + * @param res Respuesta HTTP + */ + static async update(req: Request, res: Response) { + // Variables para almacenar datos durante el proceso + let crm_to_eda: any = {}; + let modelToExport: any = {}; + let grantedRolesAt: any = []; + let enumerator: any; + let connection: any; + + // Inicio del proceso y conexión a BD + console.time("UpdateModel"); + connection = await mariadb.createConnection(sinergiaDatabase.sinergiaConn); + console.timeLog("UpdateModel", "(Create connection)"); + + try { + // Validación inicial del modelo + await updateModel.checkSinergiaModel(connection); + console.timeLog("UpdateModel", "(Checking model)"); + + await connection + .query( + // Query que une tablas base, maestras y puente + " select `table`, label , description, visible from sda_def_tables sdt union all " + + " select distinct master_table , master_table ,master_table , 0 as visible " + + " from sda_def_enumerations sde union all " + + " select distinct bridge_table , bridge_table ,bridge_table , 0 as visible " + + " from sda_def_enumerations sde " + + " where bridge_table != '' " + ) + .then(async (rows, err1) => { + if (err1) { + console.log("Error getting tables list"); + throw err1; + } + let tables = rows; + + // Query para obtener definición de columnas + const my_query = + " select sdc.`table`, sdc.`column`,`type`,sdc.label, sdc.description, sdc.decimals, sdc.aggregations, sdc.visible, sdc.stic_type, sdc.sda_hidden " + + " FROM sda_def_columns sdc " + + " union " + + " select master_table , master_id , 'text', master_id , master_id , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " union " + + " select master_table , master_column , 'text', master_column , master_column , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " union " + + " select bridge_table , source_bridge , 'text', source_bridge , source_bridge , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " where bridge_table != '' " + + " union " + + " select bridge_table , target_bridge , 'text', target_bridge , target_bridge , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " where bridge_table != '' "; + await connection.query(my_query).then(async rows => { + let columns = rows; + + // Consulta para obtener relaciones entre tablas + await connection + .query( + ` + SELECT distinct source_table, source_column, target_table, target_column, label , 0 as direccion + FROM sda_def_relationships + union + SELECT target_table as source_table, target_column as source_column , + source_table as target_table , source_column as target_column, label as label , 1 as direccion + FROM sda_def_relationships + where source_table != target_table + union + SELECT source_table , source_column , + master_table , master_id as target_column, 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table is null or bridge_table = '' + union + SELECT master_table , master_id , + source_table , source_column , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table is null or bridge_table = '' + union + SELECT source_table , source_bridge as source_column , + bridge_table , source_bridge, 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' + union + SELECT bridge_table , source_bridge, + source_table , source_bridge as source_column , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' + union + SELECT bridge_table , target_bridge, + master_table , master_id , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' + union + SELECT master_table , master_id, + bridge_table , target_bridge , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' ` + ) + .then(async rows => { + let relations = rows; + + // Obtener usuarios (activos e inactivos) + await connection + .query( + "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE password IS NOT NULL ;" + ) + .then(async users => { + let users_crm = users; + + // Obtener roles EDA + await connection + .query( + 'select "EDA_USER_ROLE" as role, b.name, "" as user_name from sda_def_groups b union select "EDA_USER_ROLE" as role, g.name as name , g.user_name from sda_def_user_groups g; ' + ) + .then(async role => { + let roles = role; + + // Obtener permisos de grupos sobre tablas + await connection + .query( + ' select distinct a.user_name as name, a.`table`, "id" as `column`, a.`group` from sda_def_permissions a where a.`group` != "" ; ' + ) + .then(async granted => { + let fullTablePermissionsForRoles = granted; + + // Obtener enumeraciones + await connection + .query( + " select source_table , source_column , master_table, master_id, master_column, bridge_table, source_bridge, target_bridge, stic_type, info from sda_def_enumerations sde ;" + ) + .then(async enums => { + let ennumeration = enums; + + // Obtener permisos de usuarios sobre tablas + await connection + .query(" select user_name as name, `table` from sda_def_permissions ") + .then(async permi => { + let fullTablePermissionsForUsers = permi; + + await connection + .query( + " select distinct user_name as name, `table`, 'id' as `column`, group_concat( distinct `group`) as `group` from sda_def_permissions where `group` != '' group by 1,2,3 " + ) + .then(async permiCol => { + let dynamicPermisssionsForGroup = permiCol; + + // Obtener permisos dinámicos de usuarios + const query = + 'select user_name as name, `table` as tabla , `column` as columna from sda_def_permissions where stic_permission_source in ( "ACL_ALLOW_OWNER")'; + await connection.query(query).then(async customUserPermissionsValue => { + console.timeLog("UpdateModel", "(Run MariaDB queries)"); + let dynamicPermisssionsForUser = customUserPermissionsValue; + + try { + // Sincronizar usuarios y grupos + crm_to_eda = await userAndGroupsToMongo.crm_to_eda_UsersAndGroups( + users_crm, + roles + ); + console.timeLog("UpdateModel", "(Syncs users and groups)"); + } catch (e) { + console.log("Error 1", e); + res.status(500).json({ status: "ko" }); + } + + try { + // Convertir permisos CRM a modelo EDA + grantedRolesAt = await updateModel.grantedRolesToModel( + fullTablePermissionsForRoles, + tables, + fullTablePermissionsForUsers, + dynamicPermisssionsForGroup, + dynamicPermisssionsForUser + ); + console.timeLog("UpdateModel", "(Converts CRM roles to EDA)"); + } catch (e) { + console.log("Error 2", e); + res.status(500).json({ status: "ko" }); + } + + try { + // Crear modelo final + modelToExport = updateModel.createModel( + tables, + columns, + relations, + grantedRolesAt, + ennumeration, + res + ); + console.timeLog("UpdateModel", "(Creating Model)"); + } catch (e) { + console.log("Error 3", e); + res.status(500).json({ status: "ko" }); + } + }); + + connection.end(); + }); + }); + }); + }); + }); + }); + }); + }); + }); + } catch (e) { + console.log("Error : ", e); + } + } + + /** + * Verifica la existencia de columnas y tablas definidas en el modelo + * @param con Conexión a la base de datos + */ + static async checkSinergiaModel(con: any) { + // Arrays para almacenar metadatos + let tablas = []; + let columns = []; + let successfulQueries = 0; + + try { + // Obtener listado de tablas y columnas + const dataset = await con + .query("select sdc.`table` as tabla, sdc.`column` as `column` FROM sda_def_columns sdc") + .catch(err => { + console.log("Error retrieving tables to check:", err); + throw err; + }); + + // Extraer nombres únicos de tablas + tablas = [...new Set(dataset.map(item => item.tabla))]; + + // Verificar cada tabla y sus columnas + for (const tabla of tablas) { + columns = [...new Set(dataset.map(item => (tabla === item.tabla ? item.column : null)))].filter( + item => item != null + ); + + // Construir y ejecutar query de prueba + const sql = " select " + columns.toString() + " from " + tabla + " limit 1 \n"; + let nexSql = sql.replace("select ,", "select ").replace(", from", " from "); + + try { + await con.query(nexSql); + successfulQueries++; + } catch (err) { + console.log(`Error executing query for table ${tabla}:`, err); + } + } + } catch (err) { + console.log("Error in model check process:", err); + } + } + + /** + * Genera y procesa los roles del modelo EDA a partir de los permisos del CRM + * @param fullTablePermissionsForRoles Permisos de grupos sobre tablas completas + * @param crmTables Tablas del CRM + * @param fullTablePermissionsForUsers Permisos de usuarios sobre tablas completas + * @param dynamicPermisssionsForGroup Permisos dinámicos para grupos + * @param dynamicPermisssionsForUser Permisos dinámicos para usuarios + * @returns Array de roles procesados + */ + static async grantedRolesToModel( + fullTablePermissionsForRoles: any, + crmTables: any, + fullTablePermissionsForUsers: any, + dynamicPermisssionsForGroup: any, + dynamicPermisssionsForUser: any + ) { + const destGrantedRoles = []; + let gr, gr2, gr3, gr4, gr5 = {}; + + // Obtener usuarios y grupos de MongoDB + const usersFound = await User.find(); + const mongoGroups = await Group.find(); + + // Función para verificar permisos duplicados + const hasExistingFullTablePermission = (newRole: any) => { + return destGrantedRoles.some(existing => + existing.column === "fullTable" && + existing.table === newRole.table && + existing.type === newRole.type && + existing.users?.toString() === newRole.users?.toString() + ); + }; + + // Procesar permisos de grupos sobre tablas + fullTablePermissionsForRoles.forEach(line => { + let match = mongoGroups.filter(i => { + return i.name === line.group; + }); + let mongoId: String; + let mongoGroup: String; + if (match.length == 1 && line.group !== undefined) { + mongoId = match[0]._id.toString(); + mongoGroup = match[0].name.toString(); + if (line.name != null) { + // Procesar grupo convertido a usuario + const found = usersFound.find(i => i.email == line.name); + gr = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "users" + }; + + // Verificar duplicados antes de agregar + if (!hasExistingFullTablePermission(gr)) { + destGrantedRoles.push(gr); + } + } else { + gr = { + groups: [mongoId], + groupsName: [mongoGroup], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "groups" + }; + destGrantedRoles.push(gr); + } + } + }); + + // Procesar permisos de usuarios sobre tablas + fullTablePermissionsForUsers.forEach(line => { + const found = usersFound.find(i => i.email == line.name); + if (found) { + gr3 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "users" + }; + if (!hasExistingFullTablePermission(gr3)) { + destGrantedRoles.push(gr3); + } + } + }); + + // Procesar permisos dinámicos de grupos + dynamicPermisssionsForGroup.forEach(line => { + const match = mongoGroups.filter(i => { + return i.name === line.group.split(",")[0]; + }); + let mongoId: String; + if (match.length == 1 && line.group !== undefined) { + mongoId = match[0]._id.toString(); + let group_name: String = " '" + line.group + "' "; + let table_name: String = " '" + line.table + "' "; + let valueAt: String = + " select record_id from sda_def_security_group_records" + + " where `group` in ( " + + group_name.split(",").join("','") + + ") and `table` = " + + table_name; + if (line.name != null) { + // Procesar grupo convertido a usuario + const found = usersFound.find(i => i.email == line.name); + gr4 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: line.column, + dynamic: true, + global: false, + type: "users", + value: [valueAt] + }; + destGrantedRoles.push(gr4); + + let valueAt2: String = " select `id` from " + line.table + " where `assigned_user_name` = 'EDA_USER' "; + gr5 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "id", + global: false, + permission: true, + dynamic: true, + type: "users", + value: [valueAt2] + }; + destGrantedRoles.push(gr5); + } + } + }); + + // Procesar permisos dinámicos de usuarios + dynamicPermisssionsForUser.forEach(line => { + const found = usersFound.find(i => i.email == line.name); + if (found) { + let valueAt: String = + "select `" + line.columna + "` from " + line.tabla + " where `" + line.columna + "` = 'EDA_USER' "; + gr5 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.tabla, + column: line.columna, + global: false, + permission: true, + dynamic: true, + type: "users", + value: [valueAt] + }; + destGrantedRoles.push(gr5); + } + }); + + return destGrantedRoles; + } + + /** + * Crea el modelo de datos completo con tablas, columnas y relaciones + * @param tables Tablas del modelo + * @param columns Columnas de las tablas + * @param relations Relaciones entre tablas + * @param grantedRoles Roles y permisos + * @param ennumeration Enumeraciones del modelo + * @param res Respuesta HTTP + * @returns Array de tablas procesadas + */ + static createModel( + tables: any, + columns: any, + relations: any, + grantedRoles: any, + ennumeration: any, + res: any + ): string[] { + let visible = false; + + // Procesar estructura de tablas + const destTables = []; + for (let i = 0; i < tables.length; i++) { + // Determinar visibilidad + if (tables[i].visible == 1) { + visible = true; + } else { + visible = false; + } + + // Crear estructura base de tabla + var tabla = { + table_name: tables[i].table, + columns: [], + relations: [], + display_name: { + default: tables[i].label, + localized: [] + }, + description: { + default: tables[i].description, + localized: [] + }, + visible: visible, + table_granted_roles: [], + table_type: [], + tableCount: 0, + no_relations: [] + }; + + // Procesar columnas y relaciones + const destColumns: any[] = updateModel.getColumnsForTable(tables[i].table, columns, ennumeration); + tabla.columns = destColumns; + + const destRelations: any[] = updateModel.getRelations(tables[i].table, relations); + tabla.relations = destRelations; + + destTables.push(tabla); + } + + // Enviar modelo a MongoDB + this.extractJsonModelAndPushToMongo(destTables, grantedRoles, res); + + return destTables; + } + + /** + * Genera la configuración de agregaciones para las columnas + * @param aggregations String con tipos de agregación separados por comas + * @returns Array de configuraciones de agregación + */ + static getAggretations(aggregations: string) { + // Procesar tipos de agregación + const aggArray = aggregations.split(","); + const agg = []; + + // Añadir configuraciones según tipos + if (aggArray.indexOf("sum") >= 0) { + agg.push({ value: "sum", display_name: "Sum" }); + } + if (aggArray.indexOf("avg") >= 0) { + agg.push({ value: "avg", display_name: "Average" }); + } + if (aggArray.indexOf("max") >= 0) { + agg.push({ value: "max", display_name: "Maximum" }); + } + if (aggArray.indexOf("min") >= 0) { + agg.push({ value: "min", display_name: "Minimum" }); + } + if (aggArray.indexOf("count") >= 0) { + agg.push({ value: "count", display_name: "Count Values" }); + } + if (aggArray.indexOf("count_distinct") > 0) { + agg.push({ value: "count_distinct", display_name: "Distinct Values" }); + } + agg.push({ value: "none", display_name: "None" }); + return agg; + } + + /** + * Procesa y devuelve las columnas para una tabla específica + * @param table Nombre de la tabla + * @param columns Array de columnas a procesar + * @param ennumeration Configuración de enumeraciones + * @returns Array de columnas procesadas + */ + static getColumnsForTable(table: string, columns: any, ennumeration: any) { + const destColumns = []; + + // Configuración por defecto de agregación + const agg_none = [ + { + value: "none", + display_name: "none" + } + ]; + + let agg_used = {}; + let colVisible = false; + + // Procesar cada columna + for (let i = 0; i < columns.length; i++) { + let c = columns[i]; + + // Determinar agregadores según tipo + if (columns[i].type) { + agg_used = this.getAggretations(columns[i].aggregations); + } else { + agg_used = agg_none; + } + + // Determinar visibilidad + if (columns[i].visible == 1) { + colVisible = true; + } else { + colVisible = false; + } + + // Procesar columna si pertenece a la tabla + if (c.table == table) { + c = { + column_name: columns[i].column, + column_type: columns[i].type == "enumeration" ? "text" : columns[i].type, + display_name: { + default: columns[i].label, + localized: [] + }, + description: { + default: columns[i].description, + localized: [] + }, + aggregation_type: agg_used, + visible: colVisible, + minimumFractionDigits: columns[i].decimals, + column_granted_roles: [], + row_granted_roles: [], + tableCount: 0, + valueListSource: {}, + hidden: columns[i].sda_hidden + }; + + // Procesar fuente de valores si existe + const foundTable = ennumeration.filter(j => j.source_table == table); + foundTable.forEach(u => { + if (u.source_column == c.column_name) { + c.valueListSource = Enumerations.enumeration_to_column(u); + } + }); + destColumns.push(c); + } + } + return destColumns; + } + + /** + * Procesa y devuelve las relaciones para una tabla específica + * @param table Nombre de la tabla + * @param relations Array de relaciones a procesar + * @returns Array de relaciones procesadas + */ + static getRelations(table: string, relations: any) { + const destRelations = []; + + // Procesar cada relación + for (let i = 0; i < relations.length; i++) { + let r = relations[i]; + if (r.source_table == table) { + let rr = { + source_table: relations[i].source_table, + source_column: [relations[i].source_column], + target_table: relations[i].target_table, + target_column: [relations[i].target_column], + visible: true, + bridge: relations[i].label == "xx-bridge|xx-bridge" ? true : false, + display_name: { + default: relations[i].direccion === 0 ? relations[i].label.split("|")[0] : relations[i].label.split("|")[1], + localized: [] + }, + autorelation: relations[i].source_table === relations[i].target_table ? true : false + }; + destRelations.push(rr); + } + } + + return destRelations; + } + + /** + * Formatea y envía el modelo final a MongoDB + * @param tables Tablas procesadas + * @param grantedRoles Roles y permisos + * @param res Respuesta HTTP + */ + static async extractJsonModelAndPushToMongo(tables: any, grantedRoles: any, res: any) { + // Inicio del formateo JSON + console.timeLog("UpdateModel", "(Start JSON formatting)"); + + // Cargar y configurar modelo base + let main_model = await JSON.parse( + fs.readFileSync( + path.join(__dirname, '../../../config/base_datamodel.json'), + "utf-8" + ) + ); + + // Configurar conexión + main_model.ds.connection.host = sinergiaDatabase.sinergiaConn.host; + main_model.ds.connection.database = sinergiaDatabase.sinergiaConn.database; + main_model.ds.connection.port = sinergiaDatabase.sinergiaConn.port; + main_model.ds.connection.user = sinergiaDatabase.sinergiaConn.user; + main_model.ds.connection.poolLimit = sinergiaDatabase.sinergiaConn.connectionLimit; + main_model.ds.connection.password = EnCrypterService.encrypt(sinergiaDatabase.sinergiaConn.password); + + // Asignar datos del modelo + main_model.ds.model.tables = tables; + main_model.ds.metadata.model_granted_roles = await grantedRoles; + + console.timeLog("UpdateModel", "(Model configuration completed)"); + + try { + // Limpiar modelo + const cleanM = new CleanModel(); + main_model = await cleanM.cleanModel(main_model); + console.timeLog("UpdateModel", "(Model cleaning completed)"); + + // Guardar metadata + fs.writeFile(`metadata.json`, JSON.stringify(main_model), { encoding: `utf-8` }, err => { + if (err) { + throw err; + } + }); + console.timeLog("UpdateModel", "(Metadata file written)"); + + // Enviar a MongoDB + await new pushModelToMongo().pushModel(main_model, res); + res.status(200).json({ status: "ok" }); + } catch (e) { + console.log("Error :", e); + res.status(500).json({ status: "ko" }); } - }); - destColumns.push(c); - } - } - return destColumns; - } - - /** Generates and processes relations for a specific table */ - static getRelations(table: string, relations: any) { - const destRelations = []; - - for (let i = 0; i < relations.length; i++) { - let r = relations[i]; - if (r.source_table == table) { - let rr = { - source_table: relations[i].source_table, - source_column: [relations[i].source_column], - target_table: relations[i].target_table, - target_column: [relations[i].target_column], - visible: true, - bridge: relations[i].label == "xx-bridge|xx-bridge" ? true : false, - display_name: { - default: relations[i].direccion === 0 ? relations[i].label.split("|")[0] : relations[i].label.split("|")[1], - localized: [] - }, - autorelation: relations[i].source_table === relations[i].target_table ? true : false - }; - destRelations.push(rr); - } - } - - return destRelations; - } - - /** Formats and pushes the final model to MongoDB */ - static async extractJsonModelAndPushToMongo(tables: any, grantedRoles: any, res: any) { - // Format tables as JSON - let main_model = await JSON.parse(fs.readFileSync("config/base_datamodel.json", "utf-8")); - main_model.ds.connection.host = sinergiaDatabase.sinergiaConn.host; - main_model.ds.connection.database = sinergiaDatabase.sinergiaConn.database; - main_model.ds.connection.port = sinergiaDatabase.sinergiaConn.port; - main_model.ds.connection.user = sinergiaDatabase.sinergiaConn.user; - main_model.ds.connection.poolLimit = sinergiaDatabase.sinergiaConn.connectionLimit; - main_model.ds.connection.password = EnCrypterService.encrypt(sinergiaDatabase.sinergiaConn.password); - main_model.ds.model.tables = tables; - main_model.ds.metadata.model_granted_roles = await grantedRoles; - - try { - const cleanM = new CleanModel(); - main_model = await cleanM.cleanModel(main_model); - fs.writeFile(`metadata.json`, JSON.stringify(main_model), { encoding: `utf-8` }, err => { - if (err) { - throw err; - } - }); - await new pushModelToMongo().pushModel(main_model, res); - res.status(200).json({ status: "ok" }); - } catch (e) { - console.log("Error :", e); - res.status(500).json({ status: "ko" }); - } - } -} + } +} \ No newline at end of file diff --git a/eda/eda_app/package.json b/eda/eda_app/package.json index fdbf894bc..7c4fce4be 100644 --- a/eda/eda_app/package.json +++ b/eda/eda_app/package.json @@ -31,7 +31,7 @@ "@fortawesome/free-solid-svg-icons": "^5.13.0", "@fullcalendar/core": "^5.1.0", "@types/d3": "^5.16.3", - "@types/leaflet": "^1.5.13", + "@types/leaflet": "^1.9.3", "angular2gridster": "^13.0.0", "canvg": "^3.0.6", "chart.js": "3.9.1", @@ -44,7 +44,7 @@ "glob-parent": "^6.0.1", "html2canvas": "^1.0.0-rc.5", "jspdf": "^2.3.1", - "leaflet": "^1.6.0", + "leaflet": "^1.9.4", "lodash": "^4.17.21", "moment": "^2.29.3", "ng2-charts": "4.1.1",