From f1eda1eb58e29c12ee7e3d99f155e42edf122981 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 25 Nov 2024 13:02:55 +0100 Subject: [PATCH 01/23] enhancements --- .../module/updateModel/service/cleanModel.ts | 240 ++++-------------- .../updateModel/updateModel.controller.ts | 13 +- 2 files changed, 68 insertions(+), 185 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/cleanModel.ts b/eda/eda_api/lib/module/updateModel/service/cleanModel.ts index 0c71d6e44..b33d1ae29 100644 --- a/eda/eda_api/lib/module/updateModel/service/cleanModel.ts +++ b/eda/eda_api/lib/module/updateModel/service/cleanModel.ts @@ -1,195 +1,67 @@ import _ from "lodash"; -import Group, { IGroup } from '../../admin/groups/model/group.model' -import DataSourceSchema from '../../datasource/model/datasource.model' +import Group, { IGroup } from '../../admin/groups/model/group.model'; +import DataSourceSchema from '../../datasource/model/datasource.model'; export class CleanModel { - - public async cleanModel(main_model : any) : Promise { - - let roles = _.cloneDeep(main_model.ds.metadata.model_granted_roles); - - let model = { - "users": [], - "usersName": [], - "none": "", - "table": "", - "column": "", - "global": "", - "permission": "", - "type": "", - "value":[] - } - - let modelAc = { - "users": [], - "usersName": [], - "none": "", - "table": "", - "column": "", - "global": "", - "permission": "", - "type": "", - "value":[] - } - - let groupModel = { - "groups": [], - "groupsName": [], - "none": "", - "table": "", - "column": "", - "global": "", - "permission": "", - "type": "", - "value":[] - } - - let model_granted_roles = [] ; - - for (let i=0;i r.table == roles[i].table - && r.column == roles[i].column - && r.type == roles[i].type - && r.global == roles[i].global - && r.none == roles[i].none - && r.permission == roles[i].permission - && r?.dynamic == roles[i]?.dynamic - ); - if( _.isEmpty(match) == false ){ - if(roles[i].value && match.value ){ - roles[i].value.forEach((e,i)=> { - if( e != match.value[i]){ - match = false; - } - }); - } - } - - - if (_.isEmpty(match) == false && roles[i].type == "users") { - if (!match.users.includes(roles[i].users[0])) {match.users.push(roles[i].users[0]) } ; - if (!match.usersName.includes(roles[i].usersName[0])) {match.usersName.push(roles[i].usersName[0]) } ; - modelAc = match; - - } else if (_.isEmpty(match) == false && roles[i].type == "groups") { - if (!match.groups.includes(roles[i].groups[0])) {match.groups.push(roles[i].groups[0]) } ; - if (!match.groupsName.includes(roles[i].groupsName[0])) {match.groupsName.push(roles[i].groupsName[0]) } ; - groupModel = match; - } - else { - if (_.isEmpty(modelAc.table) == false) {model_granted_roles.push(modelAc) }; - if (_.isEmpty(groupModel.table) == false ) {model_granted_roles.push(groupModel)}; - if (roles[i].type == "groups") { - groupModel = roles[i]; - if (_.isEmpty(groupModel.table) == false) {model_granted_roles.push(groupModel)} ; - } else { - model = roles[i]; - if (_.isEmpty(model.table) == false) {model_granted_roles.push(model)} ; - } - } + public async cleanModel(main_model: any): Promise { + const roles = _.cloneDeep(main_model.ds.metadata.model_granted_roles); + const model_granted_roles: any[] = []; + const mapRoles = new Map(); + + const addOrUpdateRole = (role: any, key: string) => { + const existingRole = mapRoles.get(key); + if (existingRole) { + if (role.type === "users") { + existingRole.users = Array.from(new Set([...existingRole.users, ...role.users])); + existingRole.usersName = Array.from(new Set([...existingRole.usersName, ...role.usersName])); + } else if (role.type === "groups") { + existingRole.groups = Array.from(new Set([...existingRole.groups, ...role.groups])); + existingRole.groupsName = Array.from(new Set([...existingRole.groupsName, ...role.groupsName])); } + } else { + mapRoles.set(key, _.cloneDeep(role)); } + }; + + roles.forEach((role: any) => { + const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.dynamic}-${role.value?.join(',')}`; + addOrUpdateRole(role, key); + }); + + mapRoles.forEach((value) => model_granted_roles.push(value)); + + // Recuperar permisos adicionales de Mongo + const finder = await DataSourceSchema.find({ _id: "111111111111111111111111" }); + const mgsmap = finder + .map((e) => e.ds.metadata.model_granted_roles) + .reduce((acc, val) => acc.concat(val), []); + // Combinar permisos únicos + const filterUniqueRoles = (roles: any[], comparator: (a: any, b: any) => boolean) => { + const seen = new Map(); + return roles.filter((role) => { + const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.users?.join(',')}-${role.groups?.join(',')}`; + if (seen.has(key)) { + return false; + } + seen.set(key, role); + return true; + }); + }; + const uniqueRoles = filterUniqueRoles(model_granted_roles, _.isEqual); - //recuperamos los model_granted_roles de mongo, donde se han añadido permisos para SCRM_* - const finder = await DataSourceSchema.find({_id: "111111111111111111111111" }) ; - let mgs = []; - const mgsmap = _.cloneDeep(finder.map(e => mgs = e.ds.metadata.model_granted_roles)); - - function objetosIgualesGrupos(objetoA: any, objetoB: any): boolean { - if (objetoA.groups != undefined && objetoB.groups != undefined - ) return ( - objetoA.groups.join(',') === objetoB.groups.join(',') && - objetoA.groupsName.join(',') === objetoB.groupsName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type - ); - } - - - function objetosIgualesUsuarios(objetoA: any, objetoB: any): boolean { - if (objetoA.users != undefined && objetoB.users != undefined && objetoA?.dynamic && objetoB?.dynamic && objetoA?.value && objetoB?.value ){ - return ( - objetoA.users.join(',') === objetoB.users.join(',') && - objetoA.usersName.join(',') === objetoB.usersName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type && - objetoA?.dynamic === objetoB?.dynamic && - objetoA?.value[0] === objetoB?.value[0] - ); - }else if (objetoA.users != undefined && objetoB.users != undefined && objetoA?.value && objetoB?.value ){ - return ( - objetoA.users.join(',') === objetoB.users.join(',') && - objetoA.usersName.join(',') === objetoB.usersName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type && - objetoA?.value[0] === objetoB?.value[0] - ); - }else if (objetoA.users != undefined && objetoB.users != undefined ){ - return ( - objetoA.users.join(',') === objetoB.users.join(',') && - objetoA.usersName.join(',') === objetoB.usersName.join(',') && - objetoA.none === objetoB.none && - objetoA.table === objetoB.table && - objetoA.column === objetoB.column && - objetoA.global === objetoB.global && - objetoA.permission === objetoB.permission && - objetoA.type === objetoB.type - ); - } - } - - - // Filtrar objetos únicos grupos - const objetosUnicosGrupos = model_granted_roles.filter((objeto, index, self) => - self.findIndex(other => objetosIgualesGrupos(objeto, other)) === index - ); - - - - // Filtrar objetos únicos usuarios - const objetosUnicosUsuarios = model_granted_roles.filter((objeto, index, self) => - self.findIndex(other => objetosIgualesUsuarios(objeto, other)) === index - ); - - model_granted_roles = objetosUnicosGrupos.concat(objetosUnicosUsuarios); + // Marcar origen de permisos + uniqueRoles.forEach((role) => (role.source = "update_model")); - model_granted_roles.forEach( r=> { - r.source = 'update_model'; - } + if (mgsmap.length) { + const userRoles = mgsmap.filter( + (r: any) => r?.source === "SDA" && !r.groupsName.some((name: string) => name.startsWith("SCRM_")) ); - - // Recuperando los permisos provenientes de SinergiaCRM - // la propiedad source --> "EDA" indica que el permiso proviene de la applicacion y no de la base de datos - if(mgsmap.length!==0) { - const userRoles = mgsmap[0].filter( (r:any) => { - return r?.source === 'SDA' && !r.groupsName.find( e => e.startsWith('SCRM_')) - }); - - // Agregando los permisos agregados previamente en la aplicacion. - const all_roles = [ ...model_granted_roles, ...userRoles]; - main_model.ds.metadata.model_granted_roles = all_roles; - } - - return main_model; + main_model.ds.metadata.model_granted_roles = [...uniqueRoles, ...userRoles]; + } else { + main_model.ds.metadata.model_granted_roles = uniqueRoles; } - } + return main_model; + } +} diff --git a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts index c355cd72d..37df313ad 100644 --- a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts +++ b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts @@ -567,6 +567,8 @@ export class updateModel { /** Formats and pushes the final model to MongoDB */ static async extractJsonModelAndPushToMongo(tables: any, grantedRoles: any, res: any) { // Format tables as JSON + console.timeLog("UpdateModel", "(Start JSON formatting)"); // AÑADIR AQUÍ - Línea antes de let main_model + let main_model = await JSON.parse(fs.readFileSync("config/base_datamodel.json", "utf-8")); main_model.ds.connection.host = sinergiaDatabase.sinergiaConn.host; main_model.ds.connection.database = sinergiaDatabase.sinergiaConn.database; @@ -577,19 +579,28 @@ export class updateModel { main_model.ds.model.tables = tables; main_model.ds.metadata.model_granted_roles = await grantedRoles; + console.timeLog("UpdateModel", "(Model configuration completed)"); // AÑADIR AQUÍ - Después de asignar todos los valores + try { const cleanM = new CleanModel(); + main_model = await cleanM.cleanModel(main_model); + console.timeLog("UpdateModel", "(Model cleaning completed)"); // AÑADIR AQUÍ - Después de cleanModel + fs.writeFile(`metadata.json`, JSON.stringify(main_model), { encoding: `utf-8` }, err => { if (err) { throw err; } }); + console.timeLog("UpdateModel", "(Metadata file written)"); // AÑADIR AQUÍ - Después de writeFile + await new pushModelToMongo().pushModel(main_model, res); + // No es necesario añadir timeLog aquí ya que pushModel ya tiene su propio console.timeEnd + res.status(200).json({ status: "ok" }); } catch (e) { console.log("Error :", e); res.status(500).json({ status: "ko" }); } - } +} } From 075a650db7ea7f745249af6b8fec716f5a8311bd Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 25 Nov 2024 15:35:10 +0100 Subject: [PATCH 02/23] controller messages --- .../service/usersAndGroupsToMongo.ts | 414 +++++++++--------- 1 file changed, 196 insertions(+), 218 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index f965e412e..0c9f793e5 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -26,241 +26,219 @@ * MongoDB database and relies on user and role data structures from SinergiaCRM. */ -import mongoose, { - connections, - Model, - mongo, - Mongoose, - QueryOptions -} from 'mongoose' -import User, { IUser } from '../../admin/users/model/user.model' -import Group, { IGroup } from '../../admin/groups/model/group.model' +import mongoose, { connections, Model, mongo, Mongoose, QueryOptions } from "mongoose"; +import User, { IUser } from "../../admin/users/model/user.model"; +import Group, { IGroup } from "../../admin/groups/model/group.model"; -mongoose.set('useFindAndModify', false); +mongoose.set("useFindAndModify", false); export class userAndGroupsToMongo { static async crm_to_eda_UsersAndGroups(users: any, roles: any) { - // Sync users and groups from CRM to EDA - let mongoUsers = await User.find() - - // Initialize users - for (let i = 0; i < users.length; i++) { - let existe = mongoUsers.find(e => e.email == users[i].email) - if (!existe) { - let user = new User({ - name: users[i].name, - email: users[i].email, - password: users[i].password, - role: [] - }) - try { - await user.save() - } catch (err) { - console.log( - 'usuario ' + - user.name + - ' (Could not insert into the MongoDB database.)' ) - } - } else { - await User.findOneAndUpdate({ name: users[i].name }, { password: users[i].password }); + console.time("Total usersAndGroupsToMongo"); + console.log(`Starting sync: ${users.length} users and ${roles.length} role assignments`); + + // Sync users and groups from CRM to EDA + console.time("Initial User Query"); + let mongoUsers = await User.find(); + console.timeEnd("Initial User Query"); + console.log(`Found ${mongoUsers.length} existing users in MongoDB`); + + // Initialize users + console.time("User Initialization"); + let usersCreated = 0; + let usersUpdated = 0; + for (let i = 0; i < users.length; i++) { + let existe = mongoUsers.find(e => e.email == users[i].email); + if (!existe) { + let user = new User({ + name: users[i].name, + email: users[i].email, + password: users[i].password, + role: [] + }); + try { + await user.save(); + usersCreated++; + } catch (err) { + console.log('usuario ' + user.name + ' (Could not insert into the MongoDB database.)'); + } + } else { + await User.findOneAndUpdate({ name: users[i].name }, { password: users[i].password }); + usersUpdated++; + } } - } - - // Initialize groups - let mongoGroups = await Group.find() - mongoUsers = await User.find() - const unique_groups = [...new Set(roles.map(item => item.name))] - - for (let i = 0; i < unique_groups.length; i++) { - let existe = mongoGroups.find(e => e.name == unique_groups[i]) - if ( - !existe && - unique_groups[i] != 'EDA_ADMIN' && - unique_groups[i] != 'EDA_RO' && - unique_groups[i] != 'EDA_DATASOURCE_CREATOR' - ) { - let group = new Group({ - role: 'EDA_USER_ROLE', - name: unique_groups[i], - users: [] - }) - try { - await group.save() - console.log(`Group ${group.name} inserted successfully`) - } catch (err) { - console.log(`Group ${group.name} already exists, skipped`) - } + console.timeEnd("User Initialization"); + console.log(`Users processed: ${usersCreated} created, ${usersUpdated} updated`); + + // Initialize groups + console.time("Group Initialization"); + let mongoGroups = await Group.find(); + mongoUsers = await User.find(); + const unique_groups = [...new Set(roles.map(item => item.name))]; + + let groupsCreated = 0; + for (let i = 0; i < unique_groups.length; i++) { + let existe = mongoGroups.find(e => e.name == unique_groups[i]); + if (!existe && + unique_groups[i] != 'EDA_ADMIN' && + unique_groups[i] != 'EDA_RO' && + unique_groups[i] != 'EDA_DATASOURCE_CREATOR' + ) { + let group = new Group({ + role: 'EDA_USER_ROLE', + name: unique_groups[i], + users: [] + }); + try { + await group.save(); + groupsCreated++; + } catch (err) { + console.log(`Group ${group.name} already exists, skipped`); + } + } } - } + console.timeEnd("Group Initialization"); + console.log(`Groups: ${mongoGroups.length} existing, ${unique_groups.length} unique, ${groupsCreated} created`); - // Synchronize users and groups - await this.syncronizeUsersGroups(mongoUsers, mongoGroups, users, roles); + // Synchronize users and groups + console.time("User-Group Synchronization"); + const stats = await this.syncronizeUsersGroups(mongoUsers, mongoGroups, users, roles); + console.timeEnd("User-Group Synchronization"); + console.log('Synchronization completed:', stats); + + console.timeEnd("Total usersAndGroupsToMongo"); } - - static async syncronizeUsersGroups( - mongoUsers: any, - mongoGroups: any, - crmUsers: any, - crmRoles: any - ) { - // Remove inactive users from CRM - mongoUsers.forEach(a => { - let existe = crmUsers.find(u => u.email === a.email); - if (existe) { - if ( - a.email !== 'eda@sinergiada.org' && - a.email !== 'eda@jortilles.com' && - a.email !== 'edaanonim@jortilles.com' && - existe.active == 0) { - User.deleteOne({ email: a.email }) - .then(function () { - }) - .catch(function (error) { - console.log('Error deleting user:', a.email, 'Details:', error); - }) - } - } - }) - // Remove deleted CRM groups - mongoGroups.forEach(a => { - if( a.name.startsWith('SCRM_') ){ - let existe = crmRoles.find(u => u.name === a.name); - if(!existe){ - Group.deleteOne( {name: a.name} ).then( function(){ console.log( a.name + ' deleted')}) - } + static async syncronizeUsersGroups(mongoUsers: any, mongoGroups: any, crmUsers: any, crmRoles: any) { + let stats = { + inactiveUsersRemoved: 0, + deletedGroups: 0, + groupsUpdated: 0, + usersRolesUpdated: 0, + groupsSaved: 0, + usersSaved: 0 + }; + + // Remove inactive users + console.time("Remove Inactive Users"); + for (const a of mongoUsers) { + let existe = crmUsers.find(u => u.email === a.email); + if (existe && existe.active == 0 && + !['eda@sinergiada.org', 'eda@jortilles.com', 'edaanonim@jortilles.com'].includes(a.email)) { + try { + await User.deleteOne({ email: a.email }); + stats.inactiveUsersRemoved++; + } catch (error) { + console.log('Error deleting user:', a.email); + } + } } - }); - - // Helper functions to check user existence in CRM - const userExistsInCRM = (user, crmUsers) => { - return crmUsers.some(crmUser => crmUser.email === user.email && crmUser.active == 1); - }; - - const userExistedInCRM = (user, crmUsers) => { - return crmUsers.some(crmUser => crmUser.email === user.email); - }; - - // Synchronize groups and users - await mongoGroups.forEach(async (group) => { - if (group.name.startsWith('SCRM_')) { - // For SCRM_ groups, maintain SDA users and sync with CRM - const crmUsersInGroup = crmRoles - .filter(role => role.name === group.name) - .map(role => mongoUsers.find(u => u.email === role.user_name)) - .filter(user => user) - .map(user => user._id); - - group.users = [ - ...group.users.filter(userId => { - const user = mongoUsers.find(u => u._id.toString() === userId.toString()); - return user && !userExistsInCRM( user, crmUsers); - }), - ...crmUsersInGroup - ]; - - // Add new CRM users to the group - const newCrmUsersInGroup = crmRoles - .filter(role => role.name === group.name) - .map(role => mongoUsers.find(u => u.email === role.user_name && userExistsInCRM(u, crmUsers) )) - .filter(user => user && !group.users.includes(user._id)) - .map(user => user._id); - - group.users = [...group.users, ...newCrmUsersInGroup]; - - } else { - // For non-SCRM_ groups, maintain SDA users and update CRM users - group.users = group.users.filter(userId => { - const user = mongoUsers.find(u => u._id.toString() === userId.toString()); - if (!user) return false; - if (userExistedInCRM(user, crmUsers)) { - return userExistsInCRM(user, crmUsers) ; + console.timeEnd("Remove Inactive Users"); + + // Remove deleted CRM groups + console.time("Process Groups"); + for (const group of mongoGroups) { + if (group.name.startsWith('SCRM_')) { + let existe = crmRoles.find(u => u.name === group.name); + if (!existe) { + try { + await Group.deleteOne({ name: group.name }); + stats.deletedGroups++; + } catch (error) { + console.log('Error deleting group:', group.name); + } + } + + // Process SCRM groups + const crmUsersInGroup = crmRoles + .filter(role => role.name === group.name) + .map(role => mongoUsers.find(u => u.email === role.user_name)) + .filter(user => user) + .map(user => user._id); + + group.users = [ + ...group.users.filter(userId => { + const user = mongoUsers.find(u => u._id.toString() === userId.toString()); + return user && !userExistsInCRM(user, crmUsers); + }), + ...crmUsersInGroup + ]; + + const newCrmUsersInGroup = crmRoles + .filter(role => role.name === group.name) + .map(role => mongoUsers.find(u => u.email === role.user_name && userExistsInCRM(u, crmUsers))) + .filter(user => user && !group.users.includes(user._id)) + .map(user => user._id); + + group.users = [...new Set([...group.users, ...newCrmUsersInGroup])]; + stats.groupsUpdated++; } else { - return true; + group.users = group.users.filter(userId => { + const user = mongoUsers.find(u => u._id.toString() === userId.toString()); + if (!user) return false; + return userExistedInCRM(user, crmUsers) ? userExistsInCRM(user, crmUsers) : true; + }); } - }); } - }); - - // Update user roles - await mongoUsers.forEach(async (user) => { - if( userExistsInCRM( user, crmUsers)) { - // Update CRM users, maintain non-SCRM_ roles - const nonCRMRoles = user.role.filter(roleId => { - const group = mongoGroups.find(g => g._id.toString() === roleId.toString() && !g.name.startsWith('SCRM_') ); - return group; - }); - - // Add CRM roles - const crmRolesForUser = crmRoles - .filter(role => role.user_name === user.email) - .map(role => mongoGroups.find(g => g.name === role.name)) - .filter(group => group) - .map(group => group._id); - - user.role = [...new Set([...nonCRMRoles, ...crmRolesForUser])]; + console.timeEnd("Process Groups"); + + // Update user roles + console.time("Update Roles"); + for (const user of mongoUsers) { + if (userExistsInCRM(user, crmUsers)) { + const nonCRMRoles = user.role.filter(roleId => { + const group = mongoGroups.find(g => g._id.toString() === roleId.toString() && !g.name.startsWith('SCRM_')); + return group; + }); + + const crmRolesForUser = crmRoles + .filter(role => role.user_name === user.email) + .map(role => mongoGroups.find(g => g.name === role.name)) + .filter(group => group) + .map(group => group._id); + + user.role = [...new Set([...nonCRMRoles, ...crmRolesForUser])]; + stats.usersRolesUpdated++; + } } - }); - - // Add admin user to EDA_ADMIN group - await mongoGroups.find(i => i.name === 'EDA_ADMIN').users.push('135792467811111111111111') - let user = await mongoUsers.find(i => i.email === ('eda@jortilles.com') ) ; - if(user){ - user.role.push('135792467811111111111110'); - }else{ - user = await mongoUsers.find(i => i.email === ('eda@sinergiada.org' ) ) - if(user){ - user.role.push('135792467811111111111110'); - }else{ - console.log('Error: Failed to assign admin role to user'); + console.timeEnd("Update Roles"); + + // Save all changes + console.time("Save Changes"); + for (const group of mongoGroups) { + try { + await Group.updateOne({ name: group.name }, { $unset: { users: {} } }); + await Group.updateOne({ name: group.name }, { $addToSet: { users: group.users } }); + stats.groupsSaved++; + } catch (err) { + console.log(`Error updating group ${group.name}`); + } } - } - // Save changes to database - await mongoGroups.forEach(async r => { - try { - await Group.updateOne({ name: r.name }, { $unset: { users: {} } }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - } catch (err) { - console.log(err); - } - try { - await Group.updateOne({ name: r.name }, { $addToSet: { users: r.users } }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - } catch (err) { - console.log(err); + const newGroupsInMongo = await Group.find(); + const newGroupsIDInMongo = newGroupsInMongo.map(g => g._id.toString()); + + for (const user of mongoUsers) { + try { + user.role = user.role.filter(r => newGroupsIDInMongo.includes(r.toString())); + await User.updateOne({ email: user.email }, { $unset: { role: {} } }); + await User.updateOne({ email: user.email }, { $addToSet: { role: user.role } }); + stats.usersSaved++; + } catch (err) { + console.log(`Error updating user ${user.email}`); + } } - }) + console.timeEnd("Save Changes"); - const newGroupsInMongo = await Group.find(); - const newGroupsIDInMongo = newGroupsInMongo.map(g=>g._id.toString()); - - // Update user roles - await mongoUsers.forEach(async user => { - user.role = user.role.filter( r => newGroupsIDInMongo.includes(r.toString() ) ) - try { - await User.updateOne({ email: user.email }, { $unset : {role: {}} }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - - await User.updateOne({ email: user.email }, { $addToSet : {role: user.role} }) - .then(function () { - }) - .catch(function (error) { - console.log(error) - }) - } catch (err) {} - }) + return stats; } } + +// Helper functions +function userExistsInCRM(user: any, crmUsers: any[]) { + return crmUsers.some(crmUser => crmUser.email === user.email && crmUser.active == 1); +} + +function userExistedInCRM(user: any, crmUsers: any[]) { + return crmUsers.some(crmUser => crmUser.email === user.email); +} \ No newline at end of file From 39c08bb38a0b2f40826ce134b80df7e97a0a9611 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Wed, 27 Nov 2024 12:16:15 +0100 Subject: [PATCH 03/23] upgrade deps --- eda/eda_app/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eda/eda_app/package.json b/eda/eda_app/package.json index fdbf894bc..7c4fce4be 100644 --- a/eda/eda_app/package.json +++ b/eda/eda_app/package.json @@ -31,7 +31,7 @@ "@fortawesome/free-solid-svg-icons": "^5.13.0", "@fullcalendar/core": "^5.1.0", "@types/d3": "^5.16.3", - "@types/leaflet": "^1.5.13", + "@types/leaflet": "^1.9.3", "angular2gridster": "^13.0.0", "canvg": "^3.0.6", "chart.js": "3.9.1", @@ -44,7 +44,7 @@ "glob-parent": "^6.0.1", "html2canvas": "^1.0.0-rc.5", "jspdf": "^2.3.1", - "leaflet": "^1.6.0", + "leaflet": "^1.9.4", "lodash": "^4.17.21", "moment": "^2.29.3", "ng2-charts": "4.1.1", From 966f505f8b7c18d4a34c2295a4b1f340f9a1b195 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 2 Dec 2024 10:22:50 +0100 Subject: [PATCH 04/23] node path console --- eda/eda_api/lib/app.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/eda/eda_api/lib/app.ts b/eda/eda_api/lib/app.ts index 6037bc0bf..dd9231ab5 100644 --- a/eda/eda_api/lib/app.ts +++ b/eda/eda_api/lib/app.ts @@ -9,6 +9,7 @@ import errorMiddleware from './middleware/error.middleware'; import Router from './router'; const path = require('path'); +console.log("NODE_MODULES PATH", path.resolve(process.cwd(), 'node_modules')); const database = require('../config/database.config'); const mongoose = require('mongoose'); const compression = require('compression'); From 679bf5c1b2d9c75cc3d2f6b805a65a5298230b8f Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 2 Dec 2024 11:17:56 +0100 Subject: [PATCH 05/23] set consolelog --- eda/eda_api/lib/app.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eda/eda_api/lib/app.ts b/eda/eda_api/lib/app.ts index dd9231ab5..a09ffd3ae 100644 --- a/eda/eda_api/lib/app.ts +++ b/eda/eda_api/lib/app.ts @@ -9,7 +9,8 @@ import errorMiddleware from './middleware/error.middleware'; import Router from './router'; const path = require('path'); -console.log("NODE_MODULES PATH", path.resolve(process.cwd(), 'node_modules')); +/* SDA CUSTOM*/ // Show current node_modules folder +/* SDA CUSTOM*/ console.log("NODE MODULES PATH",require.resolve('lodash')); const database = require('../config/database.config'); const mongoose = require('mongoose'); const compression = require('compression'); From 36ac62132cc01c83c4b4ddaa48989c3373eb58f9 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 2 Dec 2024 17:53:27 +0100 Subject: [PATCH 06/23] nuew users & group --- .../service/usersAndGroupsToMongo.ts | 487 +++++++++--------- 1 file changed, 254 insertions(+), 233 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index 0c9f793e5..f2538bd10 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -1,244 +1,265 @@ -/** - * User and Group Synchronization Service - * Part of the updateModel module - * - * This service handles the synchronization of users and groups between SinergiaCRM - * and SinergiaDA (Sinergia Data Analytics) systems. It is a crucial component of - * the updateModel process, ensuring that user and group data is consistently - * maintained across both platforms. Key functions include: - * - * 1. Importing users from SinergiaCRM to SinergiaDA, creating or updating as needed. - * 2. Importing and creating groups from SinergiaCRM roles in SinergiaDA. - * 3. Synchronizing user-group relationships based on CRM roles and SDA-specific rules. - * 4. Handling special cases for admin users and SinergiaCRM-specific groups (prefixed with 'SCRM_'). - * 5. Removing inactive users and deleted groups to maintain system consistency. - * 6. Updating the MongoDB database in SinergiaDA with the synchronized data. - * - * The userAndGroupsToMongo class contains two primary methods: - * - crm_to_eda_UsersAndGroups: Initiates the synchronization process. - * - syncronizeUsersGroups: Performs detailed synchronization logic. - * - * This service is a critical part of maintaining data integrity between SinergiaCRM - * and SinergiaDA, ensuring that analytics and reporting in SinergiaDA accurately - * reflect the current state of users and groups in the CRM system. - * - * Note: This service assumes the existence of User and Group models in the SinergiaDA - * MongoDB database and relies on user and role data structures from SinergiaCRM. - */ - import mongoose, { connections, Model, mongo, Mongoose, QueryOptions } from "mongoose"; import User, { IUser } from "../../admin/users/model/user.model"; import Group, { IGroup } from "../../admin/groups/model/group.model"; mongoose.set("useFindAndModify", false); -export class userAndGroupsToMongo { - static async crm_to_eda_UsersAndGroups(users: any, roles: any) { - console.time("Total usersAndGroupsToMongo"); - console.log(`Starting sync: ${users.length} users and ${roles.length} role assignments`); - - // Sync users and groups from CRM to EDA - console.time("Initial User Query"); - let mongoUsers = await User.find(); - console.timeEnd("Initial User Query"); - console.log(`Found ${mongoUsers.length} existing users in MongoDB`); - - // Initialize users - console.time("User Initialization"); - let usersCreated = 0; - let usersUpdated = 0; - for (let i = 0; i < users.length; i++) { - let existe = mongoUsers.find(e => e.email == users[i].email); - if (!existe) { - let user = new User({ - name: users[i].name, - email: users[i].email, - password: users[i].password, - role: [] - }); - try { - await user.save(); - usersCreated++; - } catch (err) { - console.log('usuario ' + user.name + ' (Could not insert into the MongoDB database.)'); - } - } else { - await User.findOneAndUpdate({ name: users[i].name }, { password: users[i].password }); - usersUpdated++; - } - } - console.timeEnd("User Initialization"); - console.log(`Users processed: ${usersCreated} created, ${usersUpdated} updated`); - - // Initialize groups - console.time("Group Initialization"); - let mongoGroups = await Group.find(); - mongoUsers = await User.find(); - const unique_groups = [...new Set(roles.map(item => item.name))]; - - let groupsCreated = 0; - for (let i = 0; i < unique_groups.length; i++) { - let existe = mongoGroups.find(e => e.name == unique_groups[i]); - if (!existe && - unique_groups[i] != 'EDA_ADMIN' && - unique_groups[i] != 'EDA_RO' && - unique_groups[i] != 'EDA_DATASOURCE_CREATOR' - ) { - let group = new Group({ - role: 'EDA_USER_ROLE', - name: unique_groups[i], - users: [] - }); - try { - await group.save(); - groupsCreated++; - } catch (err) { - console.log(`Group ${group.name} already exists, skipped`); - } - } - } - console.timeEnd("Group Initialization"); - console.log(`Groups: ${mongoGroups.length} existing, ${unique_groups.length} unique, ${groupsCreated} created`); - - // Synchronize users and groups - console.time("User-Group Synchronization"); - const stats = await this.syncronizeUsersGroups(mongoUsers, mongoGroups, users, roles); - console.timeEnd("User-Group Synchronization"); - console.log('Synchronization completed:', stats); - - console.timeEnd("Total usersAndGroupsToMongo"); - } - - static async syncronizeUsersGroups(mongoUsers: any, mongoGroups: any, crmUsers: any, crmRoles: any) { - let stats = { - inactiveUsersRemoved: 0, - deletedGroups: 0, - groupsUpdated: 0, - usersRolesUpdated: 0, - groupsSaved: 0, - usersSaved: 0 - }; - - // Remove inactive users - console.time("Remove Inactive Users"); - for (const a of mongoUsers) { - let existe = crmUsers.find(u => u.email === a.email); - if (existe && existe.active == 0 && - !['eda@sinergiada.org', 'eda@jortilles.com', 'edaanonim@jortilles.com'].includes(a.email)) { - try { - await User.deleteOne({ email: a.email }); - stats.inactiveUsersRemoved++; - } catch (error) { - console.log('Error deleting user:', a.email); - } - } - } - console.timeEnd("Remove Inactive Users"); - - // Remove deleted CRM groups - console.time("Process Groups"); - for (const group of mongoGroups) { - if (group.name.startsWith('SCRM_')) { - let existe = crmRoles.find(u => u.name === group.name); - if (!existe) { - try { - await Group.deleteOne({ name: group.name }); - stats.deletedGroups++; - } catch (error) { - console.log('Error deleting group:', group.name); - } - } - - // Process SCRM groups - const crmUsersInGroup = crmRoles - .filter(role => role.name === group.name) - .map(role => mongoUsers.find(u => u.email === role.user_name)) - .filter(user => user) - .map(user => user._id); - - group.users = [ - ...group.users.filter(userId => { - const user = mongoUsers.find(u => u._id.toString() === userId.toString()); - return user && !userExistsInCRM(user, crmUsers); - }), - ...crmUsersInGroup - ]; - - const newCrmUsersInGroup = crmRoles - .filter(role => role.name === group.name) - .map(role => mongoUsers.find(u => u.email === role.user_name && userExistsInCRM(u, crmUsers))) - .filter(user => user && !group.users.includes(user._id)) - .map(user => user._id); - - group.users = [...new Set([...group.users, ...newCrmUsersInGroup])]; - stats.groupsUpdated++; - } else { - group.users = group.users.filter(userId => { - const user = mongoUsers.find(u => u._id.toString() === userId.toString()); - if (!user) return false; - return userExistedInCRM(user, crmUsers) ? userExistsInCRM(user, crmUsers) : true; - }); - } - } - console.timeEnd("Process Groups"); - - // Update user roles - console.time("Update Roles"); - for (const user of mongoUsers) { - if (userExistsInCRM(user, crmUsers)) { - const nonCRMRoles = user.role.filter(roleId => { - const group = mongoGroups.find(g => g._id.toString() === roleId.toString() && !g.name.startsWith('SCRM_')); - return group; - }); - - const crmRolesForUser = crmRoles - .filter(role => role.user_name === user.email) - .map(role => mongoGroups.find(g => g.name === role.name)) - .filter(group => group) - .map(group => group._id); - - user.role = [...new Set([...nonCRMRoles, ...crmRolesForUser])]; - stats.usersRolesUpdated++; - } - } - console.timeEnd("Update Roles"); - - // Save all changes - console.time("Save Changes"); - for (const group of mongoGroups) { - try { - await Group.updateOne({ name: group.name }, { $unset: { users: {} } }); - await Group.updateOne({ name: group.name }, { $addToSet: { users: group.users } }); - stats.groupsSaved++; - } catch (err) { - console.log(`Error updating group ${group.name}`); - } - } - - const newGroupsInMongo = await Group.find(); - const newGroupsIDInMongo = newGroupsInMongo.map(g => g._id.toString()); - - for (const user of mongoUsers) { - try { - user.role = user.role.filter(r => newGroupsIDInMongo.includes(r.toString())); - await User.updateOne({ email: user.email }, { $unset: { role: {} } }); - await User.updateOne({ email: user.email }, { $addToSet: { role: user.role } }); - stats.usersSaved++; - } catch (err) { - console.log(`Error updating user ${user.email}`); - } - } - console.timeEnd("Save Changes"); - - return stats; - } +interface MongoUser extends mongoose.Document { + email: string; + password: string; + name: string; + role: mongoose.Types.ObjectId[]; + active?: number; +} + +interface MongoGroup extends mongoose.Document { + _id: mongoose.Types.ObjectId; + name: string; + role: string; + users: mongoose.Types.ObjectId[]; +} + +interface CRMUser { + name: string; + email: string; + password: string; + active: number; +} + +interface CRMRole { + name: string; + user_name?: string; +} + +interface SyncResults { + inserted: number; + updated: number; + deleted: number; } -// Helper functions -function userExistsInCRM(user: any, crmUsers: any[]) { - return crmUsers.some(crmUser => crmUser.email === user.email && crmUser.active == 1); +class DataCache { + private static instance: DataCache; + private cache: Map; + private ttl: number; + + private constructor() { + this.cache = new Map(); + this.ttl = 300000; + } + + static getInstance(): DataCache { + if (!DataCache.instance) { + DataCache.instance = new DataCache(); + } + return DataCache.instance; + } + + async getUsers(): Promise { + const cacheKey = 'all_users'; + if (!this.cache.has(cacheKey)) { + const users = await User.find().lean() as unknown as MongoUser[]; + this.cache.set(cacheKey, users); + setTimeout(() => this.cache.delete(cacheKey), this.ttl); + } + return this.cache.get(cacheKey) as MongoUser[]; + } + + async getGroups(): Promise { + const cacheKey = 'all_groups'; + if (!this.cache.has(cacheKey)) { + const groups = await Group.find().lean() as unknown as MongoGroup[]; + this.cache.set(cacheKey, groups); + setTimeout(() => this.cache.delete(cacheKey), this.ttl); + } + return this.cache.get(cacheKey) as MongoGroup[]; + } } -function userExistedInCRM(user: any, crmUsers: any[]) { - return crmUsers.some(crmUser => crmUser.email === user.email); +export class userAndGroupsToMongo { + static async crm_to_eda_UsersAndGroups(users: CRMUser[], roles: CRMRole[]) { + console.time("Total usersAndGroupsToMongo"); + console.log(`Starting sync: ${users.length} users and ${roles.length} role assignments`); + + try { + await User.collection.createIndex({ email: 1 }, { unique: true }); + await Group.collection.createIndex({ name: 1 }, { unique: true }); + + const userSyncResults = await this.optimizedUserSync(users); + const groupSyncResults = await this.optimizedGroupSync(roles); + + console.log('Sync completed:', { + users: userSyncResults, + groups: groupSyncResults + }); + + console.timeEnd("Total usersAndGroupsToMongo"); + return { userSyncResults, groupSyncResults }; + } catch (error) { + console.error("Sync error:", error); + throw error; + } + } + + private static async optimizedUserSync(crmUsers: CRMUser[]): Promise { + console.time('userSync'); + + try { + const mongoUsers = await DataCache.getInstance().getUsers(); + const emailToMongoUser = new Map( + mongoUsers.map(user => [user.email, user]) + ); + + const [syncResult, deleteResult] = await Promise.all([ + this.synchronizeUsers(crmUsers, emailToMongoUser), + this.removeInactiveUsers(crmUsers) + ]); + + console.timeEnd('userSync'); + return { + inserted: syncResult?.insertedCount || 0, + updated: syncResult?.modifiedCount || 0, + deleted: deleteResult?.deletedCount || 0 + }; + } catch (error) { + console.error('User sync error:', error); + throw error; + } + } + + private static async synchronizeUsers(crmUsers: CRMUser[], emailToMongoUser: Map) { + const batchSize = 100; + const operations: any[] = []; + + for (let i = 0; i < crmUsers.length; i += batchSize) { + const batch = crmUsers.slice(i, i + batchSize); + + batch.forEach(crmUser => { + const existingUser = emailToMongoUser.get(crmUser.email); + + if (!existingUser) { + operations.push({ + insertOne: { + document: { + name: crmUser.name, + email: crmUser.email, + password: crmUser.password, + role: [] + } + } + }); + } else if (existingUser.password !== crmUser.password) { + operations.push({ + updateOne: { + filter: { email: crmUser.email }, + update: { $set: { password: crmUser.password } } + } + }); + } + }); + } + + if (operations.length > 0) { + return await User.collection.bulkWrite(operations, { ordered: false }); + } + return null; + } + + private static async removeInactiveUsers(crmUsers: CRMUser[]) { + const excludedEmails = ['eda@sinergiada.org', 'eda@jortilles.com', 'edaanonim@jortilles.com']; + const activeEmails = new Set( + crmUsers + .filter(user => user.active === 1) + .map(user => user.email) + ); + + const bulkDelete = { + deleteMany: { + filter: { + email: { + $nin: [...excludedEmails, ...Array.from(activeEmails)] + } + } + } + }; + + return await User.collection.bulkWrite([bulkDelete]); + } + + private static async optimizedGroupSync(roles: CRMRole[]) { + console.time('groupSync'); + + try { + const mongoGroups = await DataCache.getInstance().getGroups(); + const nameToMongoGroup = new Map( + mongoGroups.map(group => [group.name, group]) + ); + + const [syncResult, userAssignments] = await Promise.all([ + this.synchronizeGroups(roles, nameToMongoGroup), + this.updateGroupUsers(roles, mongoGroups) + ]); + + console.timeEnd('groupSync'); + return { + inserted: syncResult?.insertedCount || 0, + updated: syncResult?.modifiedCount || 0, + userAssignments + }; + } catch (error) { + console.error('Group sync error:', error); + throw error; + } + } + + private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map) { + const uniqueGroups = [...new Set(roles.map(item => item.name))]; + const operations: any[] = []; + + uniqueGroups.forEach(groupName => { + if (!nameToMongoGroup.has(groupName) && + !['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(groupName)) { + operations.push({ + insertOne: { + document: { + role: 'EDA_USER_ROLE', + name: groupName, + users: [] + } + } + }); + } + }); + + if (operations.length > 0) { + return await Group.collection.bulkWrite(operations, { ordered: false }); + } + return null; + } + + private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[]) { + const groupUpdates = new Map>(); + + roles.forEach(role => { + if (!groupUpdates.has(role.name)) { + groupUpdates.set(role.name, new Set()); + } + if (role.user_name) { + groupUpdates.get(role.name)?.add(role.user_name); + } + }); + + const operations = mongoGroups + .filter(group => groupUpdates.has(group.name)) + .map(group => ({ + updateOne: { + filter: { name: group.name }, + update: { $set: { users: Array.from(groupUpdates.get(group.name) || []) } } + } + })); + + if (operations.length > 0) { + return await Group.collection.bulkWrite(operations, { ordered: false }); + } + return null; + } } \ No newline at end of file From dc5298c2385fc7e88855e64e27c0131437046192 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 3 Dec 2024 07:30:20 +0100 Subject: [PATCH 07/23] wip --- .../service/usersAndGroupsToMongo.ts | 53 ++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index f2538bd10..326db137d 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -237,29 +237,32 @@ export class userAndGroupsToMongo { } private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[]) { - const groupUpdates = new Map>(); - - roles.forEach(role => { - if (!groupUpdates.has(role.name)) { - groupUpdates.set(role.name, new Set()); - } - if (role.user_name) { - groupUpdates.get(role.name)?.add(role.user_name); - } - }); - - const operations = mongoGroups - .filter(group => groupUpdates.has(group.name)) - .map(group => ({ - updateOne: { - filter: { name: group.name }, - update: { $set: { users: Array.from(groupUpdates.get(group.name) || []) } } - } - })); - - if (operations.length > 0) { - return await Group.collection.bulkWrite(operations, { ordered: false }); - } - return null; - } + const groupUpdates = new Map>(); + const usersCache = await DataCache.getInstance().getUsers(); + const emailToId = new Map(usersCache.map(u => [u.email, u._id])); + + // Agrupar usuarios por grupo usando Set para garantizar unicidad + roles.forEach(role => { + if (!groupUpdates.has(role.name)) { + groupUpdates.set(role.name, new Set()); + } + if (role.user_name && emailToId.has(role.user_name)) { + groupUpdates.get(role.name)?.add(emailToId.get(role.user_name)); + } + }); + + const operations = mongoGroups + .filter(group => groupUpdates.has(group.name)) + .map(group => ({ + updateOne: { + filter: { name: group.name }, + update: { $set: { users: [...new Set(Array.from(groupUpdates.get(group.name) || []))] } } + } + })); + + if (operations.length > 0) { + return await Group.collection.bulkWrite(operations, { ordered: false }); + } + return null; +} } \ No newline at end of file From be267f22d649c3a02ecbcace5f149c1a963f1076 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 3 Dec 2024 08:06:27 +0100 Subject: [PATCH 08/23] fix duplication --- .../updateModel/updateModel.controller.ts | 253 +++++++++--------- 1 file changed, 131 insertions(+), 122 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts index 37df313ad..39bd159ce 100644 --- a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts +++ b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts @@ -253,148 +253,157 @@ export class updateModel { fullTablePermissionsForUsers: any, dynamicPermisssionsForGroup: any, dynamicPermisssionsForUser: any - ) { +) { const destGrantedRoles = []; - let gr, - gr2, - gr3, - gr4, - gr5 = {}; + let gr, gr2, gr3, gr4, gr5 = {}; const usersFound = await User.find(); const mongoGroups = await Group.find(); + // Función auxiliar para verificar duplicados + const hasExistingFullTablePermission = (newRole: any) => { + return destGrantedRoles.some(existing => + existing.column === "fullTable" && + existing.table === newRole.table && + existing.type === newRole.type && + existing.users?.toString() === newRole.users?.toString() + ); + }; + fullTablePermissionsForRoles.forEach(line => { - let match = mongoGroups.filter(i => { - return i.name === line.group; - }); - let mongoId: String; - let mongoGroup: String; - if (match.length == 1 && line.group !== undefined) { - mongoId = match[0]._id.toString(); - mongoGroup = match[0].name.toString(); - if (line.name != null) { - // Process group converted to user - const found = usersFound.find(i => i.email == line.name); - gr = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "users" - }; - } else { - gr = { - groups: [mongoId], - groupsName: [mongoGroup], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "groups" - }; + let match = mongoGroups.filter(i => { + return i.name === line.group; + }); + let mongoId: String; + let mongoGroup: String; + if (match.length == 1 && line.group !== undefined) { + mongoId = match[0]._id.toString(); + mongoGroup = match[0].name.toString(); + if (line.name != null) { + // Process group converted to user + const found = usersFound.find(i => i.email == line.name); + gr = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "users" + }; + + // Verificar duplicados antes de agregar + if (!hasExistingFullTablePermission(gr)) { + destGrantedRoles.push(gr); + } + } else { + gr = { + groups: [mongoId], + groupsName: [mongoGroup], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "groups" + }; + destGrantedRoles.push(gr); + } } - - destGrantedRoles.push(gr); - } }); fullTablePermissionsForUsers.forEach(line => { - const found = usersFound.find(i => i.email == line.name); - if (found) { - gr3 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "users" - }; - destGrantedRoles.push(gr3); - } + const found = usersFound.find(i => i.email == line.name); + if (found) { + gr3 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "users" + }; + if (!hasExistingFullTablePermission(gr3)) { + destGrantedRoles.push(gr3); + } + } }); dynamicPermisssionsForGroup.forEach(line => { - const match = mongoGroups.filter(i => { - return i.name === line.group.split(",")[0]; - }); - let mongoId: String; - if (match.length == 1 && line.group !== undefined) { - mongoId = match[0]._id.toString(); - let group_name: String = " '" + line.group + "' "; - let table_name: String = " '" + line.table + "' "; - let valueAt: String = - " select record_id from sda_def_security_group_records" + - " where `group` in ( " + - group_name.split(",").join("','") + - ") and `table` = " + - table_name; - if (line.name != null) { - // Process group converted to user - const found = usersFound.find(i => i.email == line.name); - gr4 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: line.column, - dynamic: true, - global: false, - type: "users", - value: [valueAt] - }; - let valueAt2: String = " select `id` from " + line.table + " where `assigned_user_name` = 'EDA_USER' "; - gr5 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "id", - global: false, - permission: true, - dynamic: true, - type: "users", - value: [valueAt2] - }; - destGrantedRoles.push(gr4); - destGrantedRoles.push(gr5); - } else { - console.log("Error: Direct group permissions found - not allowed in this context"); + const match = mongoGroups.filter(i => { + return i.name === line.group.split(",")[0]; + }); + let mongoId: String; + if (match.length == 1 && line.group !== undefined) { + mongoId = match[0]._id.toString(); + let group_name: String = " '" + line.group + "' "; + let table_name: String = " '" + line.table + "' "; + let valueAt: String = + " select record_id from sda_def_security_group_records" + + " where `group` in ( " + + group_name.split(",").join("','") + + ") and `table` = " + + table_name; + if (line.name != null) { + // Process group converted to user + const found = usersFound.find(i => i.email == line.name); + gr4 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: line.column, + dynamic: true, + global: false, + type: "users", + value: [valueAt] + }; + destGrantedRoles.push(gr4); + + let valueAt2: String = " select `id` from " + line.table + " where `assigned_user_name` = 'EDA_USER' "; + gr5 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "id", + global: false, + permission: true, + dynamic: true, + type: "users", + value: [valueAt2] + }; + destGrantedRoles.push(gr5); + } } - - destGrantedRoles.push(gr4); - } }); dynamicPermisssionsForUser.forEach(line => { - const found = usersFound.find(i => i.email == line.name); - if (found) { - let valueAt: String = - "select `" + line.columna + "` from " + line.tabla + " where `" + line.columna + "` = 'EDA_USER' "; - gr5 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.tabla, - column: line.columna, - global: false, - permission: true, - dynamic: true, - type: "users", - value: [valueAt] - }; - destGrantedRoles.push(gr5); - } + const found = usersFound.find(i => i.email == line.name); + if (found) { + let valueAt: String = + "select `" + line.columna + "` from " + line.tabla + " where `" + line.columna + "` = 'EDA_USER' "; + gr5 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.tabla, + column: line.columna, + global: false, + permission: true, + dynamic: true, + type: "users", + value: [valueAt] + }; + destGrantedRoles.push(gr5); + } }); return destGrantedRoles; - } +} static createModel( tables: any, From 43391463610f2ef952421d0d10bef0293ddc4f47 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Wed, 4 Dec 2024 09:20:00 +0100 Subject: [PATCH 09/23] tune + --- .../service/usersAndGroupsToMongo.ts | 510 +++++++++++++----- .../updateModel/updateModel.controller.ts | 2 +- 2 files changed, 363 insertions(+), 149 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index 326db137d..5a9a2df47 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -2,8 +2,10 @@ import mongoose, { connections, Model, mongo, Mongoose, QueryOptions } from "mon import User, { IUser } from "../../admin/users/model/user.model"; import Group, { IGroup } from "../../admin/groups/model/group.model"; +// Desactiva la advertencia de findAndModify deprecated mongoose.set("useFindAndModify", false); +// Interfaz que define la estructura de un usuario en MongoDB interface MongoUser extends mongoose.Document { email: string; password: string; @@ -12,6 +14,7 @@ interface MongoUser extends mongoose.Document { active?: number; } +// Interfaz que define la estructura de un grupo en MongoDB interface MongoGroup extends mongoose.Document { _id: mongoose.Types.ObjectId; name: string; @@ -19,6 +22,7 @@ interface MongoGroup extends mongoose.Document { users: mongoose.Types.ObjectId[]; } +// Interfaz que define la estructura de un usuario en el CRM interface CRMUser { name: string; email: string; @@ -26,27 +30,35 @@ interface CRMUser { active: number; } +// Interfaz que define la estructura de un rol en el CRM interface CRMRole { name: string; user_name?: string; } +// Interfaz que define la estructura de los resultados de sincronización interface SyncResults { inserted: number; updated: number; deleted: number; } +/** + * Clase que implementa un sistema de caché para usuarios y grupos + * Utiliza el patrón Singleton para asegurar una única instancia + */ class DataCache { private static instance: DataCache; private cache: Map; private ttl: number; + // Constructor privado para implementar Singleton private constructor() { this.cache = new Map(); - this.ttl = 300000; + this.ttl = 300000; // TTL de 5 minutos } + // Método para obtener la instancia única del caché static getInstance(): DataCache { if (!DataCache.instance) { DataCache.instance = new DataCache(); @@ -54,6 +66,7 @@ class DataCache { return DataCache.instance; } + // Obtiene usuarios del caché o de la base de datos si no están en caché async getUsers(): Promise { const cacheKey = 'all_users'; if (!this.cache.has(cacheKey)) { @@ -64,6 +77,7 @@ class DataCache { return this.cache.get(cacheKey) as MongoUser[]; } + // Obtiene grupos del caché o de la base de datos si no están en caché async getGroups(): Promise { const cacheKey = 'all_groups'; if (!this.cache.has(cacheKey)) { @@ -75,15 +89,25 @@ class DataCache { } } +/** + * Clase principal para la sincronización de usuarios y grupos entre CRM y MongoDB + */ export class userAndGroupsToMongo { + /** + * Método principal que sincroniza usuarios y grupos entre CRM y MongoDB + * @param users Lista de usuarios del CRM + * @param roles Lista de roles del CRM + */ static async crm_to_eda_UsersAndGroups(users: CRMUser[], roles: CRMRole[]) { console.time("Total usersAndGroupsToMongo"); console.log(`Starting sync: ${users.length} users and ${roles.length} role assignments`); try { + // Crea índices únicos para email y nombre await User.collection.createIndex({ email: 1 }, { unique: true }); await Group.collection.createIndex({ name: 1 }, { unique: true }); + // Sincroniza usuarios y grupos en paralelo const userSyncResults = await this.optimizedUserSync(users); const groupSyncResults = await this.optimizedGroupSync(roles); @@ -100,169 +124,359 @@ export class userAndGroupsToMongo { } } - private static async optimizedUserSync(crmUsers: CRMUser[]): Promise { - console.time('userSync'); - - try { - const mongoUsers = await DataCache.getInstance().getUsers(); - const emailToMongoUser = new Map( - mongoUsers.map(user => [user.email, user]) - ); - - const [syncResult, deleteResult] = await Promise.all([ - this.synchronizeUsers(crmUsers, emailToMongoUser), - this.removeInactiveUsers(crmUsers) - ]); - - console.timeEnd('userSync'); - return { - inserted: syncResult?.insertedCount || 0, - updated: syncResult?.modifiedCount || 0, - deleted: deleteResult?.deletedCount || 0 - }; - } catch (error) { - console.error('User sync error:', error); - throw error; - } - } + /** + * Sincroniza los usuarios del CRM con MongoDB de manera optimizada + * @param crmUsers Array de usuarios provenientes del CRM + * @returns Objeto con el conteo de usuarios insertados, actualizados y eliminados + */ +private static async optimizedUserSync(crmUsers: CRMUser[]): Promise { + // Inicia el contador de tiempo para medir la duración de la sincronización + console.time('userSync'); + + try { + // Obtiene todos los usuarios de MongoDB usando el caché + // Si no están en caché, DataCache hará la consulta a MongoDB + const mongoUsers = await DataCache.getInstance().getUsers(); + + // Crea un Map para acceso rápido a usuarios por email + // La clave es el email y el valor es el objeto usuario completo + const emailToMongoUser = new Map( + mongoUsers.map(user => [user.email, user]) + ); + + // Ejecuta dos operaciones en paralelo usando Promise.all: + // 1. synchronizeUsers: Sincroniza los usuarios nuevos y actualiza los existentes + // 2. removeInactiveUsers: Elimina los usuarios inactivos + const [syncResult, deleteResult] = await Promise.all([ + this.synchronizeUsers(crmUsers, emailToMongoUser), + this.removeInactiveUsers(crmUsers) + ]); + + // Detiene el contador de tiempo de sincronización + console.timeEnd('userSync'); + + // Retorna un objeto con las estadísticas de la sincronización + // Si alguna operación no retornó resultados, usa 0 como valor por defecto + return { + inserted: syncResult?.insertedCount || 0, // Número de usuarios nuevos insertados + updated: syncResult?.modifiedCount || 0, // Número de usuarios actualizados + deleted: deleteResult?.deletedCount || 0 // Número de usuarios eliminados + }; + } catch (error) { + // Si ocurre algún error durante el proceso + console.error('User sync error:', error); + // Propaga el error hacia arriba para que sea manejado por el llamador + throw error; + } +} + /** + * Sincroniza los usuarios del CRM con MongoDB procesándolos en lotes + * @param crmUsers Lista de usuarios del CRM a sincronizar + * @param emailToMongoUser Map de emails a usuarios existentes en MongoDB + * @returns Resultado de las operaciones de escritura en bulk + */ +private static async synchronizeUsers(crmUsers: CRMUser[], emailToMongoUser: Map) { + // Define el tamaño del lote para procesar usuarios + // 100 usuarios por lote para optimizar las operaciones en MongoDB + const batchSize = 100; + + // Array para almacenar todas las operaciones de MongoDB a ejecutar + const operations: any[] = []; + + // Procesa los usuarios en lotes usando un bucle for + // i se incrementa en batchSize en cada iteración + for (let i = 0; i < crmUsers.length; i += batchSize) { + // Extrae un subconjunto (lote) de usuarios usando slice + // Desde el índice i hasta i + batchSize + const batch = crmUsers.slice(i, i + batchSize); + + // Procesa cada usuario del lote actual + batch.forEach(crmUser => { + // Busca si el usuario ya existe en MongoDB usando su email + const existingUser = emailToMongoUser.get(crmUser.email); + + if (!existingUser) { + // Si el usuario no existe, prepara una operación de inserción + operations.push({ + insertOne: { + document: { + name: crmUser.name, + email: crmUser.email, + password: crmUser.password, + role: [] // Roles iniciales vacíos + } + } + }); + } else if (existingUser.password !== crmUser.password) { + // Si el usuario existe y su contraseña ha cambiado + // Prepara una operación de actualización + operations.push({ + updateOne: { + filter: { email: crmUser.email }, + update: { $set: { password: crmUser.password } } + } + }); + } + // Si el usuario existe y la contraseña no ha cambiado + // no se realiza ninguna operación + }); + } - private static async synchronizeUsers(crmUsers: CRMUser[], emailToMongoUser: Map) { - const batchSize = 100; - const operations: any[] = []; - - for (let i = 0; i < crmUsers.length; i += batchSize) { - const batch = crmUsers.slice(i, i + batchSize); - - batch.forEach(crmUser => { - const existingUser = emailToMongoUser.get(crmUser.email); - - if (!existingUser) { - operations.push({ - insertOne: { - document: { - name: crmUser.name, - email: crmUser.email, - password: crmUser.password, - role: [] - } - } - }); - } else if (existingUser.password !== crmUser.password) { - operations.push({ - updateOne: { - filter: { email: crmUser.email }, - update: { $set: { password: crmUser.password } } - } - }); - } - }); - } + // Si hay operaciones pendientes, las ejecuta todas en una sola llamada + if (operations.length > 0) { + // ordered: false permite que las operaciones continúen aunque alguna falle + return await User.collection.bulkWrite(operations, { ordered: false }); + } + + // Si no hay operaciones para ejecutar, retorna null + return null; +} - if (operations.length > 0) { - return await User.collection.bulkWrite(operations, { ordered: false }); - } - return null; - } +/** + * Elimina solo los usuarios inactivos que provienen del CRM + * @param crmUsers Lista de usuarios del CRM para verificar el estado activo + */ +private static async removeInactiveUsers(crmUsers: CRMUser[]) { + // Define usuarios del sistema que nunca deben eliminarse + const excludedEmails = [ + 'eda@sinergiada.org', + 'eda@jortilles.com', + 'edaanonim@jortilles.com' + ]; + + // Crea un Set con TODOS los emails que vienen del CRM + const crmEmails = new Set(crmUsers.map(user => user.email)); + + // Crea un Set con los emails de usuarios activos del CRM + const activeEmails = new Set( + crmUsers + .filter(user => user.active === 1) + .map(user => user.email) + ); + + // Define la operación de eliminación masiva + const bulkDelete = { + deleteMany: { + filter: { + // Solo elimina si: + $and: [ + // El email está en la lista de emails del CRM + { email: { $in: Array.from(crmEmails) } }, + // Y no está en la lista de excluidos ni activos + { email: { $nin: [...excludedEmails, ...Array.from(activeEmails)] } } + ] + } + } + }; - private static async removeInactiveUsers(crmUsers: CRMUser[]) { - const excludedEmails = ['eda@sinergiada.org', 'eda@jortilles.com', 'edaanonim@jortilles.com']; - const activeEmails = new Set( - crmUsers - .filter(user => user.active === 1) - .map(user => user.email) - ); - - const bulkDelete = { - deleteMany: { - filter: { - email: { - $nin: [...excludedEmails, ...Array.from(activeEmails)] - } - } - } - }; - - return await User.collection.bulkWrite([bulkDelete]); - } + return await User.collection.bulkWrite([bulkDelete]); +} - private static async optimizedGroupSync(roles: CRMRole[]) { - console.time('groupSync'); - - try { - const mongoGroups = await DataCache.getInstance().getGroups(); - const nameToMongoGroup = new Map( - mongoGroups.map(group => [group.name, group]) - ); - - const [syncResult, userAssignments] = await Promise.all([ - this.synchronizeGroups(roles, nameToMongoGroup), - this.updateGroupUsers(roles, mongoGroups) - ]); - - console.timeEnd('groupSync'); - return { - inserted: syncResult?.insertedCount || 0, - updated: syncResult?.modifiedCount || 0, - userAssignments - }; - } catch (error) { - console.error('Group sync error:', error); - throw error; - } - } +/** + * Sincroniza los grupos del CRM con MongoDB de manera optimizada + * @param roles Lista de roles/grupos del CRM a sincronizar + * @returns Objeto con estadísticas de la sincronización + */ +private static async optimizedGroupSync(roles: CRMRole[]) { + // Inicia el temporizador para medir el tiempo de sincronización + console.time('groupSync'); + + try { + // Obtiene todos los grupos existentes en MongoDB usando el caché + // Si no están en caché, DataCache hará la consulta a MongoDB + const mongoGroups = await DataCache.getInstance().getGroups(); + + // Crea un Map para acceso rápido a grupos por nombre + // La clave es el nombre del grupo y el valor es el objeto grupo completo + const nameToMongoGroup = new Map( + mongoGroups.map(group => [group.name, group]) + ); + + // Ejecuta dos operaciones en paralelo usando Promise.all: + // 1. synchronizeGroups: Sincroniza los grupos nuevos + // 2. updateGroupUsers: Actualiza los usuarios asignados a cada grupo + const [syncResult, userAssignments] = await Promise.all([ + this.synchronizeGroups(roles, nameToMongoGroup), + this.updateGroupUsers(roles, mongoGroups) + ]); + + // Detiene el temporizador de sincronización + console.timeEnd('groupSync'); + + // Retorna las estadísticas de la sincronización + return { + inserted: syncResult?.insertedCount || 0, // Número de grupos nuevos insertados + updated: syncResult?.modifiedCount || 0, // Número de grupos actualizados + userAssignments // Resultado de las asignaciones de usuarios + }; + } catch (error) { + // Si ocurre algún error durante el proceso + console.error('Group sync error:', error); + // Propaga el error hacia arriba para que sea manejado por el llamador + throw error; + } +} - private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map) { - const uniqueGroups = [...new Set(roles.map(item => item.name))]; - const operations: any[] = []; - - uniqueGroups.forEach(groupName => { - if (!nameToMongoGroup.has(groupName) && - !['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(groupName)) { - operations.push({ - insertOne: { - document: { - role: 'EDA_USER_ROLE', - name: groupName, - users: [] - } - } - }); - } - }); - - if (operations.length > 0) { - return await Group.collection.bulkWrite(operations, { ordered: false }); - } - return null; - } +/** + * Sincroniza los grupos del CRM con MongoDB, creando los grupos que no existen + * @param roles Lista de roles/grupos del CRM + * @param nameToMongoGroup Map de nombres a grupos existentes en MongoDB + * @returns Resultado de las operaciones de escritura en bulk + */ +private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map) { + // Obtiene una lista de nombres de grupos únicos del CRM + // Set elimina duplicados y Array.from lo convierte de nuevo a array + const uniqueGroups = [...new Set(roles.map(item => item.name))]; + + // Array para almacenar las operaciones de MongoDB a ejecutar + const operations: any[] = []; + // TODO: Mantener grupos propios de SDA, solo procesar grupos SCRM + // Procesa cada nombre de grupo único + uniqueGroups.forEach(groupName => { + // Verifica si: + // 1. El grupo no existe en MongoDB (no está en el Map) + // 2. No es uno de los grupos especiales del sistema + if (!nameToMongoGroup.has(groupName) && + !['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(groupName)) { + + // Prepara la operación de inserción para el nuevo grupo + operations.push({ + insertOne: { + document: { + role: 'EDA_USER_ROLE', // Rol por defecto + name: groupName, // Nombre del grupo + users: [] // Lista inicial vacía de usuarios + } + } + }); + } + }); + + // Si hay operaciones pendientes, las ejecuta todas en una sola llamada + if (operations.length > 0) { + // ordered: false permite que las operaciones continúen aunque alguna falle + return await Group.collection.bulkWrite(operations, { ordered: false }); + } + + // Si no hay operaciones para ejecutar, retorna null + return null; +} - private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[]) { - const groupUpdates = new Map>(); +/** +* Actualiza las relaciones bidireccionales entre usuarios y grupos +* - En los documentos de grupos: actualiza el array 'users' con los IDs de sus usuarios +* - En los documentos de usuarios: actualiza el array 'role' con los IDs de sus grupos +* +* @param roles Lista de roles provenientes del CRM con la estructura {name: string, user_name?: string} +* @param mongoGroups Lista de grupos de MongoDB con sus usuarios actuales +* @returns Resultado de las operaciones de escritura en bulk +*/ +private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[]) { + // Mapas para almacenar las actualizaciones pendientes + const groupUpdates = new Map>(); // grupos -> usuarios + const userUpdates = new Map>(); // usuarios -> grupos + + // Obtener usuarios del caché y crear mapa de email -> ID para búsqueda rápida const usersCache = await DataCache.getInstance().getUsers(); const emailToId = new Map(usersCache.map(u => [u.email, u._id])); - - // Agrupar usuarios por grupo usando Set para garantizar unicidad + + // Crear mapa de nombres de grupo -> ID para referencia rápida + const groupIdByName = new Map(mongoGroups.map(g => [g.name, g._id])); + + // PASO 1: Crear mapa de asignaciones usuario-grupo del CRM + const crmUsersInGroups = new Map>(); roles.forEach(role => { - if (!groupUpdates.has(role.name)) { - groupUpdates.set(role.name, new Set()); + if (role.name.startsWith('SCRM_') && role.user_name) { + if (!crmUsersInGroups.has(role.name)) { + crmUsersInGroups.set(role.name, new Set()); + } + crmUsersInGroups.get(role.name)?.add(role.user_name); } - if (role.user_name && emailToId.has(role.user_name)) { - groupUpdates.get(role.name)?.add(emailToId.get(role.user_name)); + }); + + // PASO 2: Procesar cada grupo de MongoDB + mongoGroups.forEach(group => { + if (group.name.startsWith('SCRM_')) { + // Para grupos del CRM (prefijo SCRM_) + const updatedUsers = new Set(); + + // 2.1: Añadir usuarios que vienen del CRM para este grupo + const crmUsersForGroup = crmUsersInGroups.get(group.name) || new Set(); + crmUsersForGroup.forEach(email => { + if (emailToId.has(email)) { + const userId = emailToId.get(email); + updatedUsers.add(userId); + + // Actualizar también los roles del usuario + if (!userUpdates.has(userId.toString())) { + userUpdates.set(userId.toString(), new Set()); + } + userUpdates.get(userId.toString())?.add(group._id); + } + }); + + // 2.2: Mantener usuarios existentes que NO son del CRM + group.users.forEach(userId => { + const user = usersCache.find(u => u._id.toString() === userId.toString()); + if (user && !crmUsersInGroups.get(group.name)?.has(user.email)) { + updatedUsers.add(userId); + + // Mantener el rol en el usuario + if (!userUpdates.has(userId.toString())) { + userUpdates.set(userId.toString(), new Set()); + } + userUpdates.get(userId.toString())?.add(group._id); + } + }); + + groupUpdates.set(group.name, updatedUsers); + } else { + // Para grupos que NO son del CRM, mantener configuración actual + groupUpdates.set(group.name, new Set(group.users)); + + // Mantener roles en usuarios para grupos no-CRM + group.users.forEach(userId => { + if (!userUpdates.has(userId.toString())) { + userUpdates.set(userId.toString(), new Set()); + } + userUpdates.get(userId.toString())?.add(group._id); + }); } }); - - const operations = mongoGroups + + // PASO 3: Preparar operaciones de actualización para grupos + const groupOperations = mongoGroups .filter(group => groupUpdates.has(group.name)) .map(group => ({ updateOne: { filter: { name: group.name }, - update: { $set: { users: [...new Set(Array.from(groupUpdates.get(group.name) || []))] } } + update: { + $set: { + users: [...Array.from(groupUpdates.get(group.name) || [])] + } + } } })); - - if (operations.length > 0) { - return await Group.collection.bulkWrite(operations, { ordered: false }); - } - return null; -} + + // PASO 4: Preparar operaciones de actualización para usuarios + const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => ({ + updateOne: { + filter: { _id: new mongoose.Types.ObjectId(userId) }, + update: { + $set: { + role: [...Array.from(groupIds)] + } + } + } + })); + + // PASO 5: Ejecutar todas las operaciones en paralelo + const results = await Promise.all([ + groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, + userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null + ]); + + // Devolver resultado de las operaciones de grupos para mantener compatibilidad + return results[0]; + } } \ No newline at end of file diff --git a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts index 39bd159ce..1b400bfb4 100644 --- a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts +++ b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts @@ -114,7 +114,7 @@ export class updateModel { // Select users await connection .query( - "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE password IS NOT NULL ;" + "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE active = 1 AND password IS NOT NULL ;" ) .then(async users => { let users_crm = users; From 54e2985db6b60fb16f2596b87fe5ba03fa8a64d1 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Wed, 4 Dec 2024 09:30:06 +0100 Subject: [PATCH 10/23] datacache rebuild --- .../service/usersAndGroupsToMongo.ts | 159 ++++++++++-------- 1 file changed, 91 insertions(+), 68 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index 5a9a2df47..1a92305eb 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -45,48 +45,58 @@ interface SyncResults { /** * Clase que implementa un sistema de caché para usuarios y grupos - * Utiliza el patrón Singleton para asegurar una única instancia + * Se reconstruye en cada ejecución para asegurar datos actualizados */ class DataCache { - private static instance: DataCache; - private cache: Map; - private ttl: number; - - // Constructor privado para implementar Singleton - private constructor() { - this.cache = new Map(); - this.ttl = 300000; // TTL de 5 minutos - } + private static instance: DataCache; + private cache: Map; - // Método para obtener la instancia única del caché - static getInstance(): DataCache { - if (!DataCache.instance) { - DataCache.instance = new DataCache(); - } - return DataCache.instance; - } + private constructor() { + this.cache = new Map(); + } - // Obtiene usuarios del caché o de la base de datos si no están en caché - async getUsers(): Promise { - const cacheKey = 'all_users'; - if (!this.cache.has(cacheKey)) { - const users = await User.find().lean() as unknown as MongoUser[]; - this.cache.set(cacheKey, users); - setTimeout(() => this.cache.delete(cacheKey), this.ttl); - } - return this.cache.get(cacheKey) as MongoUser[]; - } + /** + * Obtiene la instancia única del caché y la limpia + */ + static getInstance(): DataCache { + if (!DataCache.instance) { + DataCache.instance = new DataCache(); + } + // Limpiar la caché al obtener la instancia + DataCache.instance.clearCache(); + return DataCache.instance; + } - // Obtiene grupos del caché o de la base de datos si no están en caché - async getGroups(): Promise { - const cacheKey = 'all_groups'; - if (!this.cache.has(cacheKey)) { - const groups = await Group.find().lean() as unknown as MongoGroup[]; - this.cache.set(cacheKey, groups); - setTimeout(() => this.cache.delete(cacheKey), this.ttl); - } - return this.cache.get(cacheKey) as MongoGroup[]; - } + /** + * Limpia todos los datos de la caché + */ + private clearCache(): void { + this.cache.clear(); + } + + /** + * Obtiene usuarios de MongoDB y los almacena en caché + */ + async getUsers(): Promise { + const cacheKey = 'all_users'; + if (!this.cache.has(cacheKey)) { + const users = await User.find().lean() as unknown as MongoUser[]; + this.cache.set(cacheKey, users); + } + return this.cache.get(cacheKey) as MongoUser[]; + } + + /** + * Obtiene grupos de MongoDB y los almacena en caché + */ + async getGroups(): Promise { + const cacheKey = 'all_groups'; + if (!this.cache.has(cacheKey)) { + const groups = await Group.find().lean() as unknown as MongoGroup[]; + this.cache.set(cacheKey, groups); + } + return this.cache.get(cacheKey) as MongoGroup[]; + } } /** @@ -363,26 +373,25 @@ private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map>(); // grupos -> usuarios const userUpdates = new Map>(); // usuarios -> grupos - // Obtener usuarios del caché y crear mapa de email -> ID para búsqueda rápida + // Obtener usuarios del caché y crear mapas para búsqueda rápida const usersCache = await DataCache.getInstance().getUsers(); const emailToId = new Map(usersCache.map(u => [u.email, u._id])); - - // Crear mapa de nombres de grupo -> ID para referencia rápida - const groupIdByName = new Map(mongoGroups.map(g => [g.name, g._id])); - + const idToEmail = new Map(usersCache.map(u => [u._id.toString(), u.email])); + // PASO 1: Crear mapa de asignaciones usuario-grupo del CRM const crmUsersInGroups = new Map>(); roles.forEach(role => { @@ -393,13 +402,18 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ crmUsersInGroups.get(role.name)?.add(role.user_name); } }); - + + // Obtener todos los usuarios que vienen del CRM + const allCRMUsers = new Set(); + roles.forEach(role => { + if (role.user_name) allCRMUsers.add(role.user_name); + }); + // PASO 2: Procesar cada grupo de MongoDB mongoGroups.forEach(group => { if (group.name.startsWith('SCRM_')) { - // Para grupos del CRM (prefijo SCRM_) const updatedUsers = new Set(); - + // 2.1: Añadir usuarios que vienen del CRM para este grupo const crmUsersForGroup = crmUsersInGroups.get(group.name) || new Set(); crmUsersForGroup.forEach(email => { @@ -407,31 +421,41 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ const userId = emailToId.get(email); updatedUsers.add(userId); - // Actualizar también los roles del usuario + // Actualizar roles del usuario if (!userUpdates.has(userId.toString())) { userUpdates.set(userId.toString(), new Set()); } userUpdates.get(userId.toString())?.add(group._id); } }); - - // 2.2: Mantener usuarios existentes que NO son del CRM + + // 2.2: Revisar usuarios actuales del grupo group.users.forEach(userId => { - const user = usersCache.find(u => u._id.toString() === userId.toString()); - if (user && !crmUsersInGroups.get(group.name)?.has(user.email)) { + const userEmail = idToEmail.get(userId.toString()); + + // Si es un usuario del CRM, solo mantenerlo si está en crmUsersForGroup + if (userEmail && allCRMUsers.has(userEmail)) { + if (crmUsersForGroup.has(userEmail)) { + updatedUsers.add(userId); + if (!userUpdates.has(userId.toString())) { + userUpdates.set(userId.toString(), new Set()); + } + userUpdates.get(userId.toString())?.add(group._id); + } + // Si no está en crmUsersForGroup, no se añade (efectivamente eliminándolo) + } else { + // Si no es usuario del CRM, mantenerlo updatedUsers.add(userId); - - // Mantener el rol en el usuario if (!userUpdates.has(userId.toString())) { userUpdates.set(userId.toString(), new Set()); } userUpdates.get(userId.toString())?.add(group._id); } }); - + groupUpdates.set(group.name, updatedUsers); } else { - // Para grupos que NO son del CRM, mantener configuración actual + // Para grupos no-CRM, mantener configuración actual groupUpdates.set(group.name, new Set(group.users)); // Mantener roles en usuarios para grupos no-CRM @@ -443,7 +467,7 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ }); } }); - + // PASO 3: Preparar operaciones de actualización para grupos const groupOperations = mongoGroups .filter(group => groupUpdates.has(group.name)) @@ -457,7 +481,7 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ } } })); - + // PASO 4: Preparar operaciones de actualización para usuarios const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => ({ updateOne: { @@ -469,14 +493,13 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ } } })); - + // PASO 5: Ejecutar todas las operaciones en paralelo const results = await Promise.all([ groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null ]); - - // Devolver resultado de las operaciones de grupos para mantener compatibilidad + return results[0]; - } +} } \ No newline at end of file From 6fba7a4f66bb04995175fca8009432ea7e10f4e0 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Wed, 4 Dec 2024 09:57:32 +0100 Subject: [PATCH 11/23] verify admin documents relationships --- .../service/usersAndGroupsToMongo.ts | 218 +++++++++++------- 1 file changed, 137 insertions(+), 81 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index 1a92305eb..a697af1cd 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -126,6 +126,10 @@ export class userAndGroupsToMongo { groups: groupSyncResults }); + // Añadir verificación de admin + await this.verifyAdminRelations(); + + console.timeEnd("Total usersAndGroupsToMongo"); return { userSyncResults, groupSyncResults }; } catch (error) { @@ -373,133 +377,185 @@ private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map>(); // grupos -> usuarios - const userUpdates = new Map>(); // usuarios -> grupos + // Mapas para almacenar actualizaciones pendientes + const groupUpdates = new Map>(); + const userUpdates = new Map>(); // Obtener usuarios del caché y crear mapas para búsqueda rápida const usersCache = await DataCache.getInstance().getUsers(); const emailToId = new Map(usersCache.map(u => [u.email, u._id])); const idToEmail = new Map(usersCache.map(u => [u._id.toString(), u.email])); - - // PASO 1: Crear mapa de asignaciones usuario-grupo del CRM - const crmUsersInGroups = new Map>(); + + // Crear mapa de asignaciones usuario-grupo del CRM + const crmUsersGroups = new Map>(); roles.forEach(role => { - if (role.name.startsWith('SCRM_') && role.user_name) { - if (!crmUsersInGroups.has(role.name)) { - crmUsersInGroups.set(role.name, new Set()); + if (role.user_name) { + if (!crmUsersGroups.has(role.user_name)) { + crmUsersGroups.set(role.user_name, new Set()); + } + if (role.name.startsWith('SCRM_')) { + crmUsersGroups.get(role.user_name)?.add(role.name); } - crmUsersInGroups.get(role.name)?.add(role.user_name); } }); - - // Obtener todos los usuarios que vienen del CRM - const allCRMUsers = new Set(); - roles.forEach(role => { - if (role.user_name) allCRMUsers.add(role.user_name); - }); - - // PASO 2: Procesar cada grupo de MongoDB + + // Procesar cada grupo de MongoDB mongoGroups.forEach(group => { + const updatedUsers = new Set(); + if (group.name.startsWith('SCRM_')) { - const updatedUsers = new Set(); - - // 2.1: Añadir usuarios que vienen del CRM para este grupo - const crmUsersForGroup = crmUsersInGroups.get(group.name) || new Set(); - crmUsersForGroup.forEach(email => { - if (emailToId.has(email)) { - const userId = emailToId.get(email); - updatedUsers.add(userId); - - // Actualizar roles del usuario - if (!userUpdates.has(userId.toString())) { - userUpdates.set(userId.toString(), new Set()); - } - userUpdates.get(userId.toString())?.add(group._id); - } - }); - - // 2.2: Revisar usuarios actuales del grupo + // Manejo de grupos SCRM_ group.users.forEach(userId => { const userEmail = idToEmail.get(userId.toString()); - - // Si es un usuario del CRM, solo mantenerlo si está en crmUsersForGroup - if (userEmail && allCRMUsers.has(userEmail)) { - if (crmUsersForGroup.has(userEmail)) { + if (userEmail) { + const userCrmGroups = crmUsersGroups.get(userEmail); + if (userCrmGroups?.has(group.name)) { + // Mantener usuario solo si aún pertenece al grupo en CRM updatedUsers.add(userId); if (!userUpdates.has(userId.toString())) { userUpdates.set(userId.toString(), new Set()); } userUpdates.get(userId.toString())?.add(group._id); } - // Si no está en crmUsersForGroup, no se añade (efectivamente eliminándolo) - } else { - // Si no es usuario del CRM, mantenerlo - updatedUsers.add(userId); - if (!userUpdates.has(userId.toString())) { - userUpdates.set(userId.toString(), new Set()); - } - userUpdates.get(userId.toString())?.add(group._id); } }); - - groupUpdates.set(group.name, updatedUsers); } else { - // Para grupos no-CRM, mantener configuración actual - groupUpdates.set(group.name, new Set(group.users)); - - // Mantener roles en usuarios para grupos no-CRM + // Mantener configuración para grupos no-SCRM group.users.forEach(userId => { + updatedUsers.add(userId); if (!userUpdates.has(userId.toString())) { userUpdates.set(userId.toString(), new Set()); } userUpdates.get(userId.toString())?.add(group._id); }); } + + // Añadir nuevos usuarios del CRM al grupo + if (group.name.startsWith('SCRM_')) { + usersCache.forEach(user => { + const userCrmGroups = crmUsersGroups.get(user.email); + if (userCrmGroups?.has(group.name)) { + updatedUsers.add(user._id); + if (!userUpdates.has(user._id.toString())) { + userUpdates.set(user._id.toString(), new Set()); + } + userUpdates.get(user._id.toString())?.add(group._id); + } + }); + } + + groupUpdates.set(group.name, updatedUsers); }); - - // PASO 3: Preparar operaciones de actualización para grupos + + // Preparar operaciones de actualización const groupOperations = mongoGroups - .filter(group => groupUpdates.has(group.name)) .map(group => ({ updateOne: { filter: { name: group.name }, update: { $set: { - users: [...Array.from(groupUpdates.get(group.name) || [])] + users: Array.from(groupUpdates.get(group.name) || []) } } } - })); - - // PASO 4: Preparar operaciones de actualización para usuarios - const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => ({ - updateOne: { - filter: { _id: new mongoose.Types.ObjectId(userId) }, - update: { - $set: { - role: [...Array.from(groupIds)] + })) + .filter(op => op.updateOne.update.$set.users.length > 0); + + const userOperations = Array.from(userUpdates.entries()) + .map(([userId, groupIds]) => ({ + updateOne: { + filter: { _id: new mongoose.Types.ObjectId(userId) }, + update: { + $set: { + role: Array.from(groupIds) + } } } - } - })); - - // PASO 5: Ejecutar todas las operaciones en paralelo + })); + + // Ejecutar actualizaciones en paralelo const results = await Promise.all([ groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null ]); - + return results[0]; + } + + + /** + * Verifica y corrige las relaciones bidireccionales entre usuarios administrativos y sus roles + * @param adminGroups - Array de nombres de grupos administrativos a verificar + * @returns Resultado de las operaciones de corrección + */ + private static async verifyAdminRelations(adminGroups: string[] = ['EDA_ADMIN', 'EDA_DATASOURCE_CREATOR']) { + const operations = { + users: [] as any[], + groups: [] as any[] + }; + + const adminMongoGroups = await Group.find({ name: { $in: adminGroups } }); + const adminUsers = await User.find({ + role: { + $in: adminMongoGroups.map(g => g._id) + } + }); + + for (const group of adminMongoGroups) { + const usersInGroup = new Set(group.users.map(id => id.toString())); + const shouldBeInGroup = new Set(adminUsers + .filter(u => u.role.includes(group._id)) + .map(u => u._id.toString())); + + if (!userAndGroupsToMongo.setsAreEqual(usersInGroup, shouldBeInGroup)) { + operations.groups.push({ + updateOne: { + filter: { _id: group._id }, + update: { $set: { users: Array.from(shouldBeInGroup).map(id => new mongoose.Types.ObjectId(id)) } } + } + }); + } + } + + for (const user of adminUsers) { + const userGroups = new Set(user.role.map(id => id.toString())); + const shouldBeInRoles = new Set(adminMongoGroups + .filter(g => g.users.includes(user._id)) + .map(g => g._id.toString())); + + if (!userAndGroupsToMongo.setsAreEqual(userGroups, shouldBeInRoles)) { + operations.users.push({ + updateOne: { + filter: { _id: user._id }, + update: { $set: { role: Array.from(shouldBeInRoles).map(id => new mongoose.Types.ObjectId(id)) } } + } + }); + } + } + + if (operations.groups.length || operations.users.length) { + return Promise.all([ + operations.groups.length ? Group.bulkWrite(operations.groups) : null, + operations.users.length ? User.bulkWrite(operations.users) : null + ]); + } + + return null; } + +/** + * Compara dos Sets para verificar si contienen los mismos elementos + */ +private static setsAreEqual(a: Set, b: Set): boolean { + return a.size === b.size && [...a].every(value => b.has(value)); +} + } \ No newline at end of file From a7bf49f082464af6a15bb4da85da2f566f5a5fa4 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Wed, 4 Dec 2024 17:28:58 +0100 Subject: [PATCH 12/23] test --- .../service/usersAndGroupsToMongo.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index a697af1cd..9a229d2cd 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -1,6 +1,7 @@ import mongoose, { connections, Model, mongo, Mongoose, QueryOptions } from "mongoose"; import User, { IUser } from "../../admin/users/model/user.model"; import Group, { IGroup } from "../../admin/groups/model/group.model"; +import { Console } from "console"; // Desactiva la advertencia de findAndModify deprecated mongoose.set("useFindAndModify", false); @@ -398,22 +399,24 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ const crmUsersGroups = new Map>(); roles.forEach(role => { if (role.user_name) { + console.log('ROLE', role) if (!crmUsersGroups.has(role.user_name)) { crmUsersGroups.set(role.user_name, new Set()); } - if (role.name.startsWith('SCRM_')) { + // Incluir tanto grupos SCRM_ como grupos administrativos + if (role.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(role.name)) { crmUsersGroups.get(role.user_name)?.add(role.name); + } } }); - // Procesar cada grupo de MongoDB mongoGroups.forEach(group => { const updatedUsers = new Set(); - - if (group.name.startsWith('SCRM_')) { + + if (group.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(group.name) ) { // Manejo de grupos SCRM_ - group.users.forEach(userId => { + group.users.forEach(userId => { const userEmail = idToEmail.get(userId.toString()); if (userEmail) { const userCrmGroups = crmUsersGroups.get(userEmail); @@ -439,11 +442,13 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ } // Añadir nuevos usuarios del CRM al grupo - if (group.name.startsWith('SCRM_')) { + if (group.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(group.name) ) { usersCache.forEach(user => { const userCrmGroups = crmUsersGroups.get(user.email); if (userCrmGroups?.has(group.name)) { updatedUsers.add(user._id); + if(group.name=='EDA_ADMIN')console.log('ESTAMOS', updatedUsers) + if(group.name=='EDA_ADMIN')console.log('ESTAMOS', userUpdates) if (!userUpdates.has(user._id.toString())) { userUpdates.set(user._id.toString(), new Set()); } @@ -455,6 +460,9 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ groupUpdates.set(group.name, updatedUsers); }); + console.log('QUE',Array.from(groupUpdates.get('EDA_ADMIN'))) + + // Preparar operaciones de actualización const groupOperations = mongoGroups .map(group => ({ @@ -480,6 +488,10 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ } } })); +// console.log('groupOperations', groupOperations) +// console.log('userOperations', userOperations) +// console.log(Object.getOwnPropertyDescriptors(groupOperations)); + // Ejecutar actualizaciones en paralelo const results = await Promise.all([ From 326eec6e1cf1979a0cbdf8b49d335e795c971b5e Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 9 Dec 2024 12:01:27 +0100 Subject: [PATCH 13/23] fix path --- .../lib/module/updateModel/updateModel.controller.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts index 1b400bfb4..7756e538a 100644 --- a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts +++ b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts @@ -6,6 +6,7 @@ import { EnCrypterService } from "../../services/encrypter/encrypter.service"; import { userAndGroupsToMongo } from "./service/usersAndGroupsToMongo"; import { Enumerations } from "./service/enumerations"; import { pushModelToMongo } from "./service/push.Model.to.Mongo"; +import path from 'path'; import fs from "fs"; import { CleanModel } from "./service/cleanModel"; @@ -578,7 +579,12 @@ export class updateModel { // Format tables as JSON console.timeLog("UpdateModel", "(Start JSON formatting)"); // AÑADIR AQUÍ - Línea antes de let main_model - let main_model = await JSON.parse(fs.readFileSync("config/base_datamodel.json", "utf-8")); + let main_model = await JSON.parse( + fs.readFileSync( + path.join(__dirname, '../../../config/base_datamodel.json'), + "utf-8" + ) + ); main_model.ds.connection.host = sinergiaDatabase.sinergiaConn.host; main_model.ds.connection.database = sinergiaDatabase.sinergiaConn.database; main_model.ds.connection.port = sinergiaDatabase.sinergiaConn.port; From bf0547a627d1e2257c16d1158ee7d092381c5657 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 9 Dec 2024 16:32:16 +0100 Subject: [PATCH 14/23] ensure eda@sinergiada.org is admin & aoid role duplicates assignement --- .../service/usersAndGroupsToMongo.ts | 182 +++++++++--------- 1 file changed, 94 insertions(+), 88 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index 9a229d2cd..e14d433c1 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -386,122 +386,106 @@ private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map>(); const userUpdates = new Map>(); - // Obtener usuarios del caché y crear mapas para búsqueda rápida const usersCache = await DataCache.getInstance().getUsers(); const emailToId = new Map(usersCache.map(u => [u.email, u._id])); const idToEmail = new Map(usersCache.map(u => [u._id.toString(), u.email])); - - // Crear mapa de asignaciones usuario-grupo del CRM + + // Crear mapa de asignaciones CRM const crmUsersGroups = new Map>(); roles.forEach(role => { if (role.user_name) { - console.log('ROLE', role) if (!crmUsersGroups.has(role.user_name)) { crmUsersGroups.set(role.user_name, new Set()); } - // Incluir tanto grupos SCRM_ como grupos administrativos if (role.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(role.name)) { crmUsersGroups.get(role.user_name)?.add(role.name); - } } }); - // Procesar cada grupo de MongoDB + + // Procesar grupos y mantener unicidad mongoGroups.forEach(group => { - const updatedUsers = new Set(); + const uniqueUsers = new Set(); - if (group.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(group.name) ) { - // Manejo de grupos SCRM_ - group.users.forEach(userId => { + if (group.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(group.name)) { + // Procesar usuarios existentes sin duplicados + [...new Set(group.users)].forEach(userId => { const userEmail = idToEmail.get(userId.toString()); - if (userEmail) { - const userCrmGroups = crmUsersGroups.get(userEmail); - if (userCrmGroups?.has(group.name)) { - // Mantener usuario solo si aún pertenece al grupo en CRM - updatedUsers.add(userId); - if (!userUpdates.has(userId.toString())) { - userUpdates.set(userId.toString(), new Set()); - } - userUpdates.get(userId.toString())?.add(group._id); - } + if (userEmail && crmUsersGroups.get(userEmail)?.has(group.name)) { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); } }); - } else { - // Mantener configuración para grupos no-SCRM - group.users.forEach(userId => { - updatedUsers.add(userId); - if (!userUpdates.has(userId.toString())) { - userUpdates.set(userId.toString(), new Set()); - } - userUpdates.get(userId.toString())?.add(group._id); - }); - } - - // Añadir nuevos usuarios del CRM al grupo - if (group.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(group.name) ) { + + // Añadir nuevos usuarios únicos usersCache.forEach(user => { - const userCrmGroups = crmUsersGroups.get(user.email); - if (userCrmGroups?.has(group.name)) { - updatedUsers.add(user._id); - if(group.name=='EDA_ADMIN')console.log('ESTAMOS', updatedUsers) - if(group.name=='EDA_ADMIN')console.log('ESTAMOS', userUpdates) - if (!userUpdates.has(user._id.toString())) { - userUpdates.set(user._id.toString(), new Set()); - } - userUpdates.get(user._id.toString())?.add(group._id); + if (crmUsersGroups.get(user.email)?.has(group.name)) { + uniqueUsers.add(user._id); + this.updateUserRoles(userUpdates, user._id.toString(), group._id); } }); + } else { + // Mantener usuarios únicos para grupos no-SCRM + [...new Set(group.users)].forEach(userId => { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); } - - groupUpdates.set(group.name, updatedUsers); + + groupUpdates.set(group.name, uniqueUsers); }); - - console.log('QUE',Array.from(groupUpdates.get('EDA_ADMIN'))) - - - // Preparar operaciones de actualización - const groupOperations = mongoGroups - .map(group => ({ - updateOne: { - filter: { name: group.name }, - update: { - $set: { - users: Array.from(groupUpdates.get(group.name) || []) - } - } + + // Preparar operaciones bulk garantizando unicidad +const groupOperations = mongoGroups.map(group => { + const uniqueUsers = [...new Set(Array.from(groupUpdates.get(group.name) || []))]; + return { + updateOne: { + filter: { name: group.name }, + update: { + $set: { + users: uniqueUsers.filter((item, index) => + uniqueUsers.findIndex(i => i.toString() === item.toString()) === index + ) + } } - })) - .filter(op => op.updateOne.update.$set.users.length > 0); - - const userOperations = Array.from(userUpdates.entries()) - .map(([userId, groupIds]) => ({ - updateOne: { - filter: { _id: new mongoose.Types.ObjectId(userId) }, - update: { - $set: { - role: Array.from(groupIds) - } + } + }; +}).filter(op => op.updateOne.update.$set.users.length > 0); + +const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => { + const uniqueRoles = [...new Set(Array.from(groupIds))]; + return { + updateOne: { + filter: { _id: new mongoose.Types.ObjectId(userId) }, + update: { + $set: { + role: uniqueRoles.filter((item, index) => + uniqueRoles.findIndex(i => i.toString() === item.toString()) === index + ) } } - })); -// console.log('groupOperations', groupOperations) -// console.log('userOperations', userOperations) -// console.log(Object.getOwnPropertyDescriptors(groupOperations)); - - - // Ejecutar actualizaciones en paralelo - const results = await Promise.all([ + } + }; +}); + return Promise.all([ groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null ]); - - return results[0]; - } - +} + +private static updateUserRoles( + userUpdates: Map>, + userId: string, + groupId: mongoose.Types.ObjectId +): void { + if (!userUpdates.has(userId)) { + userUpdates.set(userId, new Set()); + } + userUpdates.get(userId)?.add(groupId); +} /** * Verifica y corrige las relaciones bidireccionales entre usuarios administrativos y sus roles @@ -509,6 +493,21 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ * @returns Resultado de las operaciones de corrección */ private static async verifyAdminRelations(adminGroups: string[] = ['EDA_ADMIN', 'EDA_DATASOURCE_CREATOR']) { + const ADMIN_USER_ID = '135792467811111111111111'; + const ADMIN_GROUP_ID = '135792467811111111111110'; + + // Forzar relación bidireccional del admin principal + await Promise.all([ + User.updateOne( + { _id: ADMIN_USER_ID }, + { $addToSet: { role: ADMIN_GROUP_ID } } + ), + Group.updateOne( + { _id: ADMIN_GROUP_ID }, + { $addToSet: { users: ADMIN_USER_ID } } + ) + ]); + const operations = { users: [] as any[], groups: [] as any[] @@ -527,6 +526,11 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ .filter(u => u.role.includes(group._id)) .map(u => u._id.toString())); + // Asegurar que el admin principal siempre está en EDA_ADMIN + if (group._id.toString() === ADMIN_GROUP_ID) { + shouldBeInGroup.add(ADMIN_USER_ID); + } + if (!userAndGroupsToMongo.setsAreEqual(usersInGroup, shouldBeInGroup)) { operations.groups.push({ updateOne: { @@ -543,6 +547,11 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ .filter(g => g.users.includes(user._id)) .map(g => g._id.toString())); + // Asegurar que el admin principal siempre tiene el rol EDA_ADMIN + if (user._id.toString() === ADMIN_USER_ID) { + shouldBeInRoles.add(ADMIN_GROUP_ID); + } + if (!userAndGroupsToMongo.setsAreEqual(userGroups, shouldBeInRoles)) { operations.users.push({ updateOne: { @@ -563,11 +572,8 @@ private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[ return null; } -/** - * Compara dos Sets para verificar si contienen los mismos elementos - */ private static setsAreEqual(a: Set, b: Set): boolean { - return a.size === b.size && [...a].every(value => b.has(value)); + if (a.size !== b.size) return false; + return [...a].every(value => b.has(value)); } - } \ No newline at end of file From c3108d916c6f4cee788ca2f9a5bfc47e4437b3e1 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 9 Dec 2024 17:36:21 +0100 Subject: [PATCH 15/23] all users --- eda/eda_api/lib/module/updateModel/updateModel.controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts index 7756e538a..1dd031352 100644 --- a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts +++ b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts @@ -112,10 +112,10 @@ export class updateModel { ) .then(async rows => { let relations = rows; - // Select users + // Select users (all users, not only active, neccesary to determine users to delete in SDA) await connection .query( - "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE active = 1 AND password IS NOT NULL ;" + "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE password IS NOT NULL ;" ) .then(async users => { let users_crm = users; From eab39fbd2f54ab118409747edb59a3a90754394a Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Mon, 9 Dec 2024 17:38:00 +0100 Subject: [PATCH 16/23] debug scripts --- eda/eda_api/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eda/eda_api/package.json b/eda/eda_api/package.json index 6329895a8..ef8686d27 100644 --- a/eda/eda_api/package.json +++ b/eda/eda_api/package.json @@ -5,7 +5,8 @@ "main": "index.js", "scripts": { "start": "nodemon", - "start:forever": "npm run build && forever start -c node ./dist/server.js", + "start:forever": "npm run build && forever start -c node ./dist/server.js", + "start:debug": "npm run build && forever start -c node --inspect=0.0.0.0:9229 ./dist/server.js", "start:pm2": "npm run build && pm2 start ./dist/server.js ", "edapi": "forever start -c node ./dist/server.js", "build": "tsc && npm run movejpgs", From 5441e8332063dd7beace83fa07a8e97f01e0666c Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 10 Dec 2024 10:03:11 +0100 Subject: [PATCH 17/23] intermedia --- .../service/usersAndGroupsToMongo.ts | 77 ++++++++----------- 1 file changed, 34 insertions(+), 43 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index e14d433c1..a5fe9221c 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -379,7 +379,8 @@ private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map [u.email, u._id])); const idToEmail = new Map(usersCache.map(u => [u._id.toString(), u.email])); - // Crear mapa de asignaciones CRM + // Mapa de asignaciones CRM const crmUsersGroups = new Map>(); roles.forEach(role => { - if (role.user_name) { + if (role.user_name && role.name.startsWith('SCRM_')) { if (!crmUsersGroups.has(role.user_name)) { crmUsersGroups.set(role.user_name, new Set()); } - if (role.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(role.name)) { - crmUsersGroups.get(role.user_name)?.add(role.name); - } + crmUsersGroups.get(role.user_name)?.add(role.name); } }); - // Procesar grupos y mantener unicidad mongoGroups.forEach(group => { const uniqueUsers = new Set(); - if (group.name.startsWith('SCRM_') || ['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(group.name)) { - // Procesar usuarios existentes sin duplicados - [...new Set(group.users)].forEach(userId => { + if (group.name === 'EDA_ADMIN') { + group.users.forEach(userId => { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + } else if (group.name.startsWith('SCRM_')) { + // Mantener los usuarios existentes que siguen en CRM + group.users.forEach(userId => { const userEmail = idToEmail.get(userId.toString()); if (userEmail && crmUsersGroups.get(userEmail)?.has(group.name)) { uniqueUsers.add(userId); - this.updateUserRoles(userUpdates, userId.toString(), group._id); } }); - // Añadir nuevos usuarios únicos + // Añadir nuevos usuarios del CRM usersCache.forEach(user => { if (crmUsersGroups.get(user.email)?.has(group.name)) { uniqueUsers.add(user._id); - this.updateUserRoles(userUpdates, user._id.toString(), group._id); } }); + + // Actualizar roles si hay usuarios + uniqueUsers.forEach(userId => { + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); } else { - // Mantener usuarios únicos para grupos no-SCRM - [...new Set(group.users)].forEach(userId => { + // Para grupos no-SCRM, mantener todos los usuarios existentes + group.users.forEach(userId => { uniqueUsers.add(userId); this.updateUserRoles(userUpdates, userId.toString(), group._id); }); } - groupUpdates.set(group.name, uniqueUsers); + if (uniqueUsers.size > 0) { + groupUpdates.set(group.name, uniqueUsers); + } }); - // Preparar operaciones bulk garantizando unicidad -const groupOperations = mongoGroups.map(group => { - const uniqueUsers = [...new Set(Array.from(groupUpdates.get(group.name) || []))]; - return { + // Preparar operaciones bulk + const groupOperations = Array.from(groupUpdates.entries()).map(([groupName, userIds]) => ({ updateOne: { - filter: { name: group.name }, - update: { - $set: { - users: uniqueUsers.filter((item, index) => - uniqueUsers.findIndex(i => i.toString() === item.toString()) === index - ) - } - } + filter: { name: groupName }, + update: { $set: { users: Array.from(userIds) } } } - }; -}).filter(op => op.updateOne.update.$set.users.length > 0); + })); -const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => { - const uniqueRoles = [...new Set(Array.from(groupIds))]; - return { + const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => ({ updateOne: { filter: { _id: new mongoose.Types.ObjectId(userId) }, - update: { - $set: { - role: uniqueRoles.filter((item, index) => - uniqueRoles.findIndex(i => i.toString() === item.toString()) === index - ) - } - } + update: { $set: { role: Array.from(groupIds) } } } - }; -}); + })); + return Promise.all([ groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null @@ -492,7 +483,7 @@ private static updateUserRoles( * @param adminGroups - Array de nombres de grupos administrativos a verificar * @returns Resultado de las operaciones de corrección */ - private static async verifyAdminRelations(adminGroups: string[] = ['EDA_ADMIN', 'EDA_DATASOURCE_CREATOR']) { + private static async verifyAdminRelations(adminGroups: string[] = ['EDA_ADMIN']) { const ADMIN_USER_ID = '135792467811111111111111'; const ADMIN_GROUP_ID = '135792467811111111111110'; From 071f5872ba37c1af8825598d47db7fc72c1ddc62 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 10 Dec 2024 10:14:24 +0100 Subject: [PATCH 18/23] with comments --- .../service/usersAndGroupsToMongo.ts | 676 +++++++++--------- 1 file changed, 320 insertions(+), 356 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index a5fe9221c..6422b80fb 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -3,10 +3,10 @@ import User, { IUser } from "../../admin/users/model/user.model"; import Group, { IGroup } from "../../admin/groups/model/group.model"; import { Console } from "console"; -// Desactiva la advertencia de findAndModify deprecated +// Desactiva warnings obsoletos de mongoose mongoose.set("useFindAndModify", false); -// Interfaz que define la estructura de un usuario en MongoDB +// Define estructura de usuario en MongoDB interface MongoUser extends mongoose.Document { email: string; password: string; @@ -15,7 +15,7 @@ interface MongoUser extends mongoose.Document { active?: number; } -// Interfaz que define la estructura de un grupo en MongoDB +// Define estructura de grupo en MongoDB interface MongoGroup extends mongoose.Document { _id: mongoose.Types.ObjectId; name: string; @@ -23,7 +23,7 @@ interface MongoGroup extends mongoose.Document { users: mongoose.Types.ObjectId[]; } -// Interfaz que define la estructura de un usuario en el CRM +// Define estructura de usuario en CRM interface CRMUser { name: string; email: string; @@ -31,13 +31,13 @@ interface CRMUser { active: number; } -// Interfaz que define la estructura de un rol en el CRM +// Define estructura de rol en CRM interface CRMRole { name: string; user_name?: string; } -// Interfaz que define la estructura de los resultados de sincronización +// Define estructura de resultados de sincronización interface SyncResults { inserted: number; updated: number; @@ -45,8 +45,8 @@ interface SyncResults { } /** - * Clase que implementa un sistema de caché para usuarios y grupos - * Se reconstruye en cada ejecución para asegurar datos actualizados + * Implementa un sistema de caché para usuarios y grupos en MongoDB + * Se reconstruye en cada ejecución para mantener datos actualizados */ class DataCache { private static instance: DataCache; @@ -57,26 +57,27 @@ class DataCache { } /** - * Obtiene la instancia única del caché y la limpia + * Obtiene o crea la instancia única del caché + * @returns Instancia del caché limpia */ static getInstance(): DataCache { if (!DataCache.instance) { DataCache.instance = new DataCache(); } - // Limpiar la caché al obtener la instancia DataCache.instance.clearCache(); return DataCache.instance; } /** - * Limpia todos los datos de la caché + * Elimina todos los datos almacenados en caché */ private clearCache(): void { this.cache.clear(); } /** - * Obtiene usuarios de MongoDB y los almacena en caché + * Recupera usuarios de MongoDB usando caché + * @returns Lista de usuarios */ async getUsers(): Promise { const cacheKey = 'all_users'; @@ -88,7 +89,8 @@ class DataCache { } /** - * Obtiene grupos de MongoDB y los almacena en caché + * Recupera grupos de MongoDB usando caché + * @returns Lista de grupos */ async getGroups(): Promise { const cacheKey = 'all_groups'; @@ -100,394 +102,344 @@ class DataCache { } } -/** - * Clase principal para la sincronización de usuarios y grupos entre CRM y MongoDB - */ export class userAndGroupsToMongo { - /** - * Método principal que sincroniza usuarios y grupos entre CRM y MongoDB - * @param users Lista de usuarios del CRM - * @param roles Lista de roles del CRM - */ - static async crm_to_eda_UsersAndGroups(users: CRMUser[], roles: CRMRole[]) { - console.time("Total usersAndGroupsToMongo"); - console.log(`Starting sync: ${users.length} users and ${roles.length} role assignments`); - - try { - // Crea índices únicos para email y nombre - await User.collection.createIndex({ email: 1 }, { unique: true }); - await Group.collection.createIndex({ name: 1 }, { unique: true }); - - // Sincroniza usuarios y grupos en paralelo - const userSyncResults = await this.optimizedUserSync(users); - const groupSyncResults = await this.optimizedGroupSync(roles); - - console.log('Sync completed:', { - users: userSyncResults, - groups: groupSyncResults - }); - - // Añadir verificación de admin - await this.verifyAdminRelations(); - - - console.timeEnd("Total usersAndGroupsToMongo"); - return { userSyncResults, groupSyncResults }; - } catch (error) { - console.error("Sync error:", error); - throw error; - } - } - - /** - * Sincroniza los usuarios del CRM con MongoDB de manera optimizada - * @param crmUsers Array de usuarios provenientes del CRM - * @returns Objeto con el conteo de usuarios insertados, actualizados y eliminados - */ -private static async optimizedUserSync(crmUsers: CRMUser[]): Promise { - // Inicia el contador de tiempo para medir la duración de la sincronización - console.time('userSync'); - - try { - // Obtiene todos los usuarios de MongoDB usando el caché - // Si no están en caché, DataCache hará la consulta a MongoDB - const mongoUsers = await DataCache.getInstance().getUsers(); - - // Crea un Map para acceso rápido a usuarios por email - // La clave es el email y el valor es el objeto usuario completo - const emailToMongoUser = new Map( - mongoUsers.map(user => [user.email, user]) - ); - - // Ejecuta dos operaciones en paralelo usando Promise.all: - // 1. synchronizeUsers: Sincroniza los usuarios nuevos y actualiza los existentes - // 2. removeInactiveUsers: Elimina los usuarios inactivos - const [syncResult, deleteResult] = await Promise.all([ - this.synchronizeUsers(crmUsers, emailToMongoUser), - this.removeInactiveUsers(crmUsers) - ]); + /** + * Coordina la sincronización completa de usuarios y grupos entre CRM y MongoDB + * Gestiona la creación de índices y ejecuta las sincronizaciones en paralelo + * + * @param users Lista de usuarios del CRM a sincronizar + * @param roles Lista de roles/grupos del CRM a sincronizar + * @returns Resultados de la sincronización + */ + static async crm_to_eda_UsersAndGroups(users: CRMUser[], roles: CRMRole[]) { + console.time("Total usersAndGroupsToMongo"); + console.log(`Starting sync: ${users.length} users and ${roles.length} role assignments`); - // Detiene el contador de tiempo de sincronización - console.timeEnd('userSync'); + try { + // Asegura índices únicos + await User.collection.createIndex({ email: 1 }, { unique: true }); + await Group.collection.createIndex({ name: 1 }, { unique: true }); - // Retorna un objeto con las estadísticas de la sincronización - // Si alguna operación no retornó resultados, usa 0 como valor por defecto - return { - inserted: syncResult?.insertedCount || 0, // Número de usuarios nuevos insertados - updated: syncResult?.modifiedCount || 0, // Número de usuarios actualizados - deleted: deleteResult?.deletedCount || 0 // Número de usuarios eliminados - }; - } catch (error) { - // Si ocurre algún error durante el proceso - console.error('User sync error:', error); - // Propaga el error hacia arriba para que sea manejado por el llamador - throw error; + const userSyncResults = await this.optimizedUserSync(users); + const groupSyncResults = await this.optimizedGroupSync(roles); + + console.log('Sync completed:', { + users: userSyncResults, + groups: groupSyncResults + }); + + await this.verifyAdminRelations(); + + console.timeEnd("Total usersAndGroupsToMongo"); + return { userSyncResults, groupSyncResults }; + } catch (error) { + console.error("Sync error:", error); + throw error; + } } -} - /** - * Sincroniza los usuarios del CRM con MongoDB procesándolos en lotes - * @param crmUsers Lista de usuarios del CRM a sincronizar - * @param emailToMongoUser Map de emails a usuarios existentes en MongoDB - * @returns Resultado de las operaciones de escritura en bulk - */ -private static async synchronizeUsers(crmUsers: CRMUser[], emailToMongoUser: Map) { - // Define el tamaño del lote para procesar usuarios - // 100 usuarios por lote para optimizar las operaciones en MongoDB - const batchSize = 100; - - // Array para almacenar todas las operaciones de MongoDB a ejecutar - const operations: any[] = []; - - // Procesa los usuarios en lotes usando un bucle for - // i se incrementa en batchSize en cada iteración - for (let i = 0; i < crmUsers.length; i += batchSize) { - // Extrae un subconjunto (lote) de usuarios usando slice - // Desde el índice i hasta i + batchSize - const batch = crmUsers.slice(i, i + batchSize); + + /** + * Gestiona la sincronización optimizada de usuarios + * Procesa actualizaciones e inserciones en lotes + * + * @param crmUsers Lista de usuarios del CRM + * @returns Estadísticas de la sincronización + */ + private static async optimizedUserSync(crmUsers: CRMUser[]): Promise { + console.time('userSync'); - // Procesa cada usuario del lote actual - batch.forEach(crmUser => { - // Busca si el usuario ya existe en MongoDB usando su email - const existingUser = emailToMongoUser.get(crmUser.email); + try { + const mongoUsers = await DataCache.getInstance().getUsers(); + const emailToMongoUser = new Map( + mongoUsers.map(user => [user.email, user]) + ); - if (!existingUser) { - // Si el usuario no existe, prepara una operación de inserción - operations.push({ - insertOne: { - document: { - name: crmUser.name, - email: crmUser.email, - password: crmUser.password, - role: [] // Roles iniciales vacíos - } - } - }); - } else if (existingUser.password !== crmUser.password) { - // Si el usuario existe y su contraseña ha cambiado - // Prepara una operación de actualización - operations.push({ - updateOne: { - filter: { email: crmUser.email }, - update: { $set: { password: crmUser.password } } - } - }); - } - // Si el usuario existe y la contraseña no ha cambiado - // no se realiza ninguna operación - }); + const [syncResult, deleteResult] = await Promise.all([ + this.synchronizeUsers(crmUsers, emailToMongoUser), + this.removeInactiveUsers(crmUsers) + ]); + + console.timeEnd('userSync'); + + return { + inserted: syncResult?.insertedCount || 0, + updated: syncResult?.modifiedCount || 0, + deleted: deleteResult?.deletedCount || 0 + }; + } catch (error) { + console.error('User sync error:', error); + throw error; + } } - // Si hay operaciones pendientes, las ejecuta todas en una sola llamada - if (operations.length > 0) { - // ordered: false permite que las operaciones continúen aunque alguna falle - return await User.collection.bulkWrite(operations, { ordered: false }); - } - - // Si no hay operaciones para ejecutar, retorna null - return null; -} + /** + * Procesa usuarios en lotes para sincronización + * Maneja creación de nuevos usuarios y actualización de contraseñas + * + * @param crmUsers Lista de usuarios a sincronizar + * @param emailToMongoUser Mapa de usuarios existentes + * @returns Resultado de operaciones bulk + */ + private static async synchronizeUsers(crmUsers: CRMUser[], emailToMongoUser: Map) { + const batchSize = 100; + const operations: any[] = []; -/** - * Elimina solo los usuarios inactivos que provienen del CRM - * @param crmUsers Lista de usuarios del CRM para verificar el estado activo - */ -private static async removeInactiveUsers(crmUsers: CRMUser[]) { - // Define usuarios del sistema que nunca deben eliminarse - const excludedEmails = [ - 'eda@sinergiada.org', - 'eda@jortilles.com', - 'edaanonim@jortilles.com' - ]; - - // Crea un Set con TODOS los emails que vienen del CRM - const crmEmails = new Set(crmUsers.map(user => user.email)); - - // Crea un Set con los emails de usuarios activos del CRM - const activeEmails = new Set( - crmUsers - .filter(user => user.active === 1) - .map(user => user.email) - ); - - // Define la operación de eliminación masiva - const bulkDelete = { - deleteMany: { - filter: { - // Solo elimina si: - $and: [ - // El email está en la lista de emails del CRM - { email: { $in: Array.from(crmEmails) } }, - // Y no está en la lista de excluidos ni activos - { email: { $nin: [...excludedEmails, ...Array.from(activeEmails)] } } - ] - } + for (let i = 0; i < crmUsers.length; i += batchSize) { + const batch = crmUsers.slice(i, i + batchSize); + + batch.forEach(crmUser => { + const existingUser = emailToMongoUser.get(crmUser.email); + + if (!existingUser) { + operations.push({ + insertOne: { + document: { + name: crmUser.name, + email: crmUser.email, + password: crmUser.password, + role: [] + } + } + }); + } else if (existingUser.password !== crmUser.password) { + operations.push({ + updateOne: { + filter: { email: crmUser.email }, + update: { $set: { password: crmUser.password } } + } + }); + } + }); } - }; - - return await User.collection.bulkWrite([bulkDelete]); -} -/** - * Sincroniza los grupos del CRM con MongoDB de manera optimizada - * @param roles Lista de roles/grupos del CRM a sincronizar - * @returns Objeto con estadísticas de la sincronización - */ -private static async optimizedGroupSync(roles: CRMRole[]) { - // Inicia el temporizador para medir el tiempo de sincronización - console.time('groupSync'); - - try { - // Obtiene todos los grupos existentes en MongoDB usando el caché - // Si no están en caché, DataCache hará la consulta a MongoDB - const mongoGroups = await DataCache.getInstance().getGroups(); - - // Crea un Map para acceso rápido a grupos por nombre - // La clave es el nombre del grupo y el valor es el objeto grupo completo - const nameToMongoGroup = new Map( - mongoGroups.map(group => [group.name, group]) - ); + if (operations.length > 0) { + return await User.collection.bulkWrite(operations, { ordered: false }); + } - // Ejecuta dos operaciones en paralelo usando Promise.all: - // 1. synchronizeGroups: Sincroniza los grupos nuevos - // 2. updateGroupUsers: Actualiza los usuarios asignados a cada grupo - const [syncResult, userAssignments] = await Promise.all([ - this.synchronizeGroups(roles, nameToMongoGroup), - this.updateGroupUsers(roles, mongoGroups) - ]); + return null; + } + - // Detiene el temporizador de sincronización - console.timeEnd('groupSync'); - // Retorna las estadísticas de la sincronización - return { - inserted: syncResult?.insertedCount || 0, // Número de grupos nuevos insertados - updated: syncResult?.modifiedCount || 0, // Número de grupos actualizados - userAssignments // Resultado de las asignaciones de usuarios + /** + * Elimina usuarios inactivos del CRM preservando usuarios del sistema + * + * @param crmUsers Lista de usuarios del CRM + * @returns Resultado de la operación de eliminación + */ + private static async removeInactiveUsers(crmUsers: CRMUser[]) { + const excludedEmails = [ + 'eda@sinergiada.org', + 'eda@jortilles.com', + 'edaanonim@jortilles.com' + ]; + + const crmEmails = new Set(crmUsers.map(user => user.email)); + const activeEmails = new Set( + crmUsers + .filter(user => user.active === 1) + .map(user => user.email) + ); + + const bulkDelete = { + deleteMany: { + filter: { + $and: [ + { email: { $in: Array.from(crmEmails) } }, + { email: { $nin: [...excludedEmails, ...Array.from(activeEmails)] } } + ] + } + } }; - } catch (error) { - // Si ocurre algún error durante el proceso - console.error('Group sync error:', error); - // Propaga el error hacia arriba para que sea manejado por el llamador - throw error; + + return await User.collection.bulkWrite([bulkDelete]); } -} -/** - * Sincroniza los grupos del CRM con MongoDB, creando los grupos que no existen - * @param roles Lista de roles/grupos del CRM - * @param nameToMongoGroup Map de nombres a grupos existentes en MongoDB - * @returns Resultado de las operaciones de escritura en bulk - */ -private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map) { - // Obtiene una lista de nombres de grupos únicos del CRM - // Set elimina duplicados y Array.from lo convierte de nuevo a array - const uniqueGroups = [...new Set(roles.map(item => item.name))]; - - // Array para almacenar las operaciones de MongoDB a ejecutar - const operations: any[] = []; - // TODO: Mantener grupos propios de SDA, solo procesar grupos SCRM - // Procesa cada nombre de grupo único - uniqueGroups.forEach(groupName => { - // Verifica si: - // 1. El grupo no existe en MongoDB (no está en el Map) - // 2. No es uno de los grupos especiales del sistema - if (!nameToMongoGroup.has(groupName) && - !['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(groupName)) { + /** + * Gestiona la sincronización optimizada de grupos + * Coordina creación de grupos y actualización de membresías + * + * @param roles Lista de roles del CRM + * @returns Estadísticas de sincronización + */ + private static async optimizedGroupSync(roles: CRMRole[]) { + console.time('groupSync'); + + try { + const mongoGroups = await DataCache.getInstance().getGroups(); + const nameToMongoGroup = new Map( + mongoGroups.map(group => [group.name, group]) + ); - // Prepara la operación de inserción para el nuevo grupo - operations.push({ - insertOne: { - document: { - role: 'EDA_USER_ROLE', // Rol por defecto - name: groupName, // Nombre del grupo - users: [] // Lista inicial vacía de usuarios - } - } - }); + const [syncResult, userAssignments] = await Promise.all([ + this.synchronizeGroups(roles, nameToMongoGroup), + this.updateGroupUsers(roles, mongoGroups) + ]); + + console.timeEnd('groupSync'); + + return { + inserted: syncResult?.insertedCount || 0, + updated: syncResult?.modifiedCount || 0, + userAssignments + }; + } catch (error) { + console.error('Group sync error:', error); + throw error; } - }); - - // Si hay operaciones pendientes, las ejecuta todas en una sola llamada - if (operations.length > 0) { - // ordered: false permite que las operaciones continúen aunque alguna falle - return await Group.collection.bulkWrite(operations, { ordered: false }); } - // Si no hay operaciones para ejecutar, retorna null - return null; -} - -/** -* Actualiza las relaciones bidireccionales entre usuarios y grupos, manejando tanto grupos SCRM_ como no-SCRM. -* Garantiza que los usuarios que ya no pertenecen a grupos SCRM_ en CRM sean removidos de estos grupos en MongoDB, -* mientras preserva los usuarios en EDA_RO independientemente de su origen. -* -* @param roles - Lista de roles del CRM con estructura {name: string, user_name?: string} -* @param mongoGroups - Lista de grupos existentes en MongoDB -* @returns Resultado de las operaciones de escritura en bulk -*/ -private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[]) { - const groupUpdates = new Map>(); - const userUpdates = new Map>(); - - const usersCache = await DataCache.getInstance().getUsers(); - const emailToId = new Map(usersCache.map(u => [u.email, u._id])); - const idToEmail = new Map(usersCache.map(u => [u._id.toString(), u.email])); - - // Mapa de asignaciones CRM - const crmUsersGroups = new Map>(); - roles.forEach(role => { - if (role.user_name && role.name.startsWith('SCRM_')) { - if (!crmUsersGroups.has(role.user_name)) { - crmUsersGroups.set(role.user_name, new Set()); + /** + * Crea nuevos grupos del CRM en MongoDB + * Excluye grupos especiales del sistema + * + * @param roles Lista de roles del CRM + * @param nameToMongoGroup Mapa de grupos existentes + * @returns Resultado de operaciones bulk + */ + private static async synchronizeGroups(roles: CRMRole[], nameToMongoGroup: Map) { + const uniqueGroups = [...new Set(roles.map(item => item.name))]; + const operations: any[] = []; + + uniqueGroups.forEach(groupName => { + if (!nameToMongoGroup.has(groupName) && + !['EDA_ADMIN', 'EDA_RO', 'EDA_DATASOURCE_CREATOR'].includes(groupName)) { + + operations.push({ + insertOne: { + document: { + role: 'EDA_USER_ROLE', + name: groupName, + users: [] + } + } + }); } - crmUsersGroups.get(role.user_name)?.add(role.name); + }); + + if (operations.length > 0) { + return await Group.collection.bulkWrite(operations, { ordered: false }); } - }); - mongoGroups.forEach(group => { - const uniqueUsers = new Set(); + return null; + } + + /** + * Actualiza las membresías de usuarios en grupos + * Mantiene grupos SCRM_ sincronizados con CRM y preserva otros grupos + * + * @param roles Lista de roles del CRM + * @param mongoGroups Lista de grupos en MongoDB + * @returns Resultado de operaciones bulk + */ + private static async updateGroupUsers(roles: CRMRole[], mongoGroups: MongoGroup[]) { + const groupUpdates = new Map>(); + const userUpdates = new Map>(); - if (group.name === 'EDA_ADMIN') { - group.users.forEach(userId => { - uniqueUsers.add(userId); - this.updateUserRoles(userUpdates, userId.toString(), group._id); - }); - } else if (group.name.startsWith('SCRM_')) { - // Mantener los usuarios existentes que siguen en CRM - group.users.forEach(userId => { - const userEmail = idToEmail.get(userId.toString()); - if (userEmail && crmUsersGroups.get(userEmail)?.has(group.name)) { - uniqueUsers.add(userId); + const usersCache = await DataCache.getInstance().getUsers(); + const emailToId = new Map(usersCache.map(u => [u.email, u._id])); + const idToEmail = new Map(usersCache.map(u => [u._id.toString(), u.email])); + + // Mapeo de asignaciones CRM + const crmUsersGroups = new Map>(); + roles.forEach(role => { + if (role.user_name && role.name.startsWith('SCRM_')) { + if (!crmUsersGroups.has(role.user_name)) { + crmUsersGroups.set(role.user_name, new Set()); } - }); + crmUsersGroups.get(role.user_name)?.add(role.name); + } + }); - // Añadir nuevos usuarios del CRM - usersCache.forEach(user => { - if (crmUsersGroups.get(user.email)?.has(group.name)) { - uniqueUsers.add(user._id); - } - }); + // Procesamiento de grupos + mongoGroups.forEach(group => { + const uniqueUsers = new Set(); + + if (group.name === 'EDA_ADMIN') { + group.users.forEach(userId => { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + } else if (group.name.startsWith('SCRM_')) { + group.users.forEach(userId => { + const userEmail = idToEmail.get(userId.toString()); + if (userEmail && crmUsersGroups.get(userEmail)?.has(group.name)) { + uniqueUsers.add(userId); + } + }); - // Actualizar roles si hay usuarios - uniqueUsers.forEach(userId => { - this.updateUserRoles(userUpdates, userId.toString(), group._id); - }); - } else { - // Para grupos no-SCRM, mantener todos los usuarios existentes - group.users.forEach(userId => { - uniqueUsers.add(userId); - this.updateUserRoles(userUpdates, userId.toString(), group._id); - }); - } + usersCache.forEach(user => { + if (crmUsersGroups.get(user.email)?.has(group.name)) { + uniqueUsers.add(user._id); + } + }); - if (uniqueUsers.size > 0) { - groupUpdates.set(group.name, uniqueUsers); - } - }); + uniqueUsers.forEach(userId => { + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + } else { + group.users.forEach(userId => { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + } - // Preparar operaciones bulk - const groupOperations = Array.from(groupUpdates.entries()).map(([groupName, userIds]) => ({ - updateOne: { - filter: { name: groupName }, - update: { $set: { users: Array.from(userIds) } } - } - })); + if (uniqueUsers.size > 0) { + groupUpdates.set(group.name, uniqueUsers); + } + }); - const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => ({ - updateOne: { - filter: { _id: new mongoose.Types.ObjectId(userId) }, - update: { $set: { role: Array.from(groupIds) } } - } - })); + // Preparación de operaciones bulk + const groupOperations = Array.from(groupUpdates.entries()).map(([groupName, userIds]) => ({ + updateOne: { + filter: { name: groupName }, + update: { $set: { users: Array.from(userIds) } } + } + })); - return Promise.all([ - groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, - userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null - ]); -} + const userOperations = Array.from(userUpdates.entries()).map(([userId, groupIds]) => ({ + updateOne: { + filter: { _id: new mongoose.Types.ObjectId(userId) }, + update: { $set: { role: Array.from(groupIds) } } + } + })); + return Promise.all([ + groupOperations.length > 0 ? Group.collection.bulkWrite(groupOperations, { ordered: false }) : null, + userOperations.length > 0 ? User.collection.bulkWrite(userOperations, { ordered: false }) : null + ]); + } + + /** + * Actualiza las relaciones de roles de usuario en el mapa de actualizaciones + * @param userUpdates - Mapa de actualizaciones usuario-roles + * @param userId - ID del usuario + * @param groupId - ID del grupo a añadir + */ private static updateUserRoles( userUpdates: Map>, userId: string, groupId: mongoose.Types.ObjectId ): void { + // Inicializar set si no existe if (!userUpdates.has(userId)) { userUpdates.set(userId, new Set()); } + // Añadir nuevo groupId al set de roles userUpdates.get(userId)?.add(groupId); } - - /** - * Verifica y corrige las relaciones bidireccionales entre usuarios administrativos y sus roles - * @param adminGroups - Array de nombres de grupos administrativos a verificar - * @returns Resultado de las operaciones de corrección + +/** + * Verifica y mantiene la integridad de las relaciones de usuarios administrativos + * Asegura que el usuario admin principal tenga los permisos correctos + * @param adminGroups - Grupos administrativos a verificar (por defecto solo EDA_ADMIN) + * @returns Resultado de las operaciones de corrección o null si no hay cambios */ - private static async verifyAdminRelations(adminGroups: string[] = ['EDA_ADMIN']) { +private static async verifyAdminRelations(adminGroups: string[] = ['EDA_ADMIN']) { const ADMIN_USER_ID = '135792467811111111111111'; const ADMIN_GROUP_ID = '135792467811111111111110'; - // Forzar relación bidireccional del admin principal + // Garantizar relación bidireccional del admin principal await Promise.all([ User.updateOne( { _id: ADMIN_USER_ID }, @@ -504,6 +456,7 @@ private static updateUserRoles( groups: [] as any[] }; + // Obtener grupos y usuarios administrativos const adminMongoGroups = await Group.find({ name: { $in: adminGroups } }); const adminUsers = await User.find({ role: { @@ -511,17 +464,19 @@ private static updateUserRoles( } }); + // Verificar y corregir pertenencia a grupos for (const group of adminMongoGroups) { const usersInGroup = new Set(group.users.map(id => id.toString())); const shouldBeInGroup = new Set(adminUsers .filter(u => u.role.includes(group._id)) .map(u => u._id.toString())); - // Asegurar que el admin principal siempre está en EDA_ADMIN + // Asegurar admin principal en EDA_ADMIN if (group._id.toString() === ADMIN_GROUP_ID) { shouldBeInGroup.add(ADMIN_USER_ID); } + // Actualizar si hay discrepancias if (!userAndGroupsToMongo.setsAreEqual(usersInGroup, shouldBeInGroup)) { operations.groups.push({ updateOne: { @@ -532,17 +487,19 @@ private static updateUserRoles( } } + // Verificar y corregir roles de usuarios for (const user of adminUsers) { const userGroups = new Set(user.role.map(id => id.toString())); const shouldBeInRoles = new Set(adminMongoGroups .filter(g => g.users.includes(user._id)) .map(g => g._id.toString())); - // Asegurar que el admin principal siempre tiene el rol EDA_ADMIN + // Asegurar rol EDA_ADMIN para admin principal if (user._id.toString() === ADMIN_USER_ID) { shouldBeInRoles.add(ADMIN_GROUP_ID); } + // Actualizar si hay discrepancias if (!userAndGroupsToMongo.setsAreEqual(userGroups, shouldBeInRoles)) { operations.users.push({ updateOne: { @@ -553,6 +510,7 @@ private static updateUserRoles( } } + // Ejecutar operaciones si existen if (operations.groups.length || operations.users.length) { return Promise.all([ operations.groups.length ? Group.bulkWrite(operations.groups) : null, @@ -563,6 +521,12 @@ private static updateUserRoles( return null; } +/** + * Compara dos Sets de strings para determinar si contienen los mismos elementos + * @param a - Primer Set a comparar + * @param b - Segundo Set a comparar + * @returns true si los Sets son iguales, false en caso contrario + */ private static setsAreEqual(a: Set, b: Set): boolean { if (a.size !== b.size) return false; return [...a].every(value => b.has(value)); From 6c2a6ac63a9cb174802db5d16504bd34c99a8cf5 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 10 Dec 2024 11:32:40 +0100 Subject: [PATCH 19/23] filtered users --- .../lib/module/updateModel/service/usersAndGroupsToMongo.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index 6422b80fb..e711fcd13 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -180,10 +180,12 @@ export class userAndGroupsToMongo { * @param emailToMongoUser Mapa de usuarios existentes * @returns Resultado de operaciones bulk */ - private static async synchronizeUsers(crmUsers: CRMUser[], emailToMongoUser: Map) { + private static async synchronizeUsers(crmUsersRaw: CRMUser[], emailToMongoUser: Map) { const batchSize = 100; const operations: any[] = []; + let crmUsers = crmUsersRaw.filter(element => element.active === 1); + for (let i = 0; i < crmUsers.length; i += batchSize) { const batch = crmUsers.slice(i, i + batchSize); From 072949e37a7d1b790bdfb5b78f68a6ccaf5733bc Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 10 Dec 2024 12:14:21 +0100 Subject: [PATCH 20/23] remove unneccesary method --- .../service/usersAndGroupsToMongo.ts | 135 ++++-------------- 1 file changed, 24 insertions(+), 111 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index e711fcd13..9afa64533 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -128,8 +128,6 @@ export class userAndGroupsToMongo { groups: groupSyncResults }); - await this.verifyAdminRelations(); - console.timeEnd("Total usersAndGroupsToMongo"); return { userSyncResults, groupSyncResults }; } catch (error) { @@ -352,17 +350,34 @@ export class userAndGroupsToMongo { crmUsersGroups.get(role.user_name)?.add(role.name); } }); - + + // Obtener usuarios que deben estar en EDA_ADMIN desde sda_def_user_groups + const edaAdminUsers = new Set( + roles.filter(role => + role.user_name && + role.name === 'EDA_ADMIN' + ).map(role => role.user_name!) + ); + // Procesamiento de grupos mongoGroups.forEach(group => { const uniqueUsers = new Set(); - if (group.name === 'EDA_ADMIN') { - group.users.forEach(userId => { - uniqueUsers.add(userId); - this.updateUserRoles(userUpdates, userId.toString(), group._id); - }); - } else if (group.name.startsWith('SCRM_')) { + if (group.name === 'EDA_ADMIN') { + // Mantener usuarios existentes + group.users.forEach(userId => { + uniqueUsers.add(userId); + this.updateUserRoles(userUpdates, userId.toString(), group._id); + }); + + // Añadir usuarios de sda_def_user_groups + usersCache.forEach(user => { + if (edaAdminUsers.has(user.email)) { + uniqueUsers.add(user._id); + this.updateUserRoles(userUpdates, user._id.toString(), group._id); + } + }); + } else if (group.name.startsWith('SCRM_') || group.name === 'EDA_ADMIN') { group.users.forEach(userId => { const userEmail = idToEmail.get(userId.toString()); if (userEmail && crmUsersGroups.get(userEmail)?.has(group.name)) { @@ -431,106 +446,4 @@ private static updateUserRoles( userUpdates.get(userId)?.add(groupId); } -/** - * Verifica y mantiene la integridad de las relaciones de usuarios administrativos - * Asegura que el usuario admin principal tenga los permisos correctos - * @param adminGroups - Grupos administrativos a verificar (por defecto solo EDA_ADMIN) - * @returns Resultado de las operaciones de corrección o null si no hay cambios - */ -private static async verifyAdminRelations(adminGroups: string[] = ['EDA_ADMIN']) { - const ADMIN_USER_ID = '135792467811111111111111'; - const ADMIN_GROUP_ID = '135792467811111111111110'; - - // Garantizar relación bidireccional del admin principal - await Promise.all([ - User.updateOne( - { _id: ADMIN_USER_ID }, - { $addToSet: { role: ADMIN_GROUP_ID } } - ), - Group.updateOne( - { _id: ADMIN_GROUP_ID }, - { $addToSet: { users: ADMIN_USER_ID } } - ) - ]); - - const operations = { - users: [] as any[], - groups: [] as any[] - }; - - // Obtener grupos y usuarios administrativos - const adminMongoGroups = await Group.find({ name: { $in: adminGroups } }); - const adminUsers = await User.find({ - role: { - $in: adminMongoGroups.map(g => g._id) - } - }); - - // Verificar y corregir pertenencia a grupos - for (const group of adminMongoGroups) { - const usersInGroup = new Set(group.users.map(id => id.toString())); - const shouldBeInGroup = new Set(adminUsers - .filter(u => u.role.includes(group._id)) - .map(u => u._id.toString())); - - // Asegurar admin principal en EDA_ADMIN - if (group._id.toString() === ADMIN_GROUP_ID) { - shouldBeInGroup.add(ADMIN_USER_ID); - } - - // Actualizar si hay discrepancias - if (!userAndGroupsToMongo.setsAreEqual(usersInGroup, shouldBeInGroup)) { - operations.groups.push({ - updateOne: { - filter: { _id: group._id }, - update: { $set: { users: Array.from(shouldBeInGroup).map(id => new mongoose.Types.ObjectId(id)) } } - } - }); - } - } - - // Verificar y corregir roles de usuarios - for (const user of adminUsers) { - const userGroups = new Set(user.role.map(id => id.toString())); - const shouldBeInRoles = new Set(adminMongoGroups - .filter(g => g.users.includes(user._id)) - .map(g => g._id.toString())); - - // Asegurar rol EDA_ADMIN para admin principal - if (user._id.toString() === ADMIN_USER_ID) { - shouldBeInRoles.add(ADMIN_GROUP_ID); - } - - // Actualizar si hay discrepancias - if (!userAndGroupsToMongo.setsAreEqual(userGroups, shouldBeInRoles)) { - operations.users.push({ - updateOne: { - filter: { _id: user._id }, - update: { $set: { role: Array.from(shouldBeInRoles).map(id => new mongoose.Types.ObjectId(id)) } } - } - }); - } - } - - // Ejecutar operaciones si existen - if (operations.groups.length || operations.users.length) { - return Promise.all([ - operations.groups.length ? Group.bulkWrite(operations.groups) : null, - operations.users.length ? User.bulkWrite(operations.users) : null - ]); - } - - return null; -} - -/** - * Compara dos Sets de strings para determinar si contienen los mismos elementos - * @param a - Primer Set a comparar - * @param b - Segundo Set a comparar - * @returns true si los Sets son iguales, false en caso contrario - */ -private static setsAreEqual(a: Set, b: Set): boolean { - if (a.size !== b.size) return false; - return [...a].every(value => b.has(value)); -} } \ No newline at end of file From 3d3fd37ec7080fee0dadde820c58aea9aa9d63ea Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 10 Dec 2024 15:30:13 +0100 Subject: [PATCH 21/23] comments --- .../module/updateModel/service/cleanModel.ts | 133 +- .../service/usersAndGroupsToMongo.ts | 23 + .../updateModel/updateModel.controller.ts | 1307 +++++++++-------- 3 files changed, 805 insertions(+), 658 deletions(-) diff --git a/eda/eda_api/lib/module/updateModel/service/cleanModel.ts b/eda/eda_api/lib/module/updateModel/service/cleanModel.ts index b33d1ae29..0754c13c1 100644 --- a/eda/eda_api/lib/module/updateModel/service/cleanModel.ts +++ b/eda/eda_api/lib/module/updateModel/service/cleanModel.ts @@ -1,67 +1,92 @@ +/** +* Este módulo se encarga de limpiar y consolidar los permisos del modelo de datos. +* Maneja la deduplicación de roles, fusión de permisos y sincronización con MongoDB. +*/ + import _ from "lodash"; import Group, { IGroup } from '../../admin/groups/model/group.model'; import DataSourceSchema from '../../datasource/model/datasource.model'; export class CleanModel { - public async cleanModel(main_model: any): Promise { - const roles = _.cloneDeep(main_model.ds.metadata.model_granted_roles); - const model_granted_roles: any[] = []; - const mapRoles = new Map(); + /** + * Limpia y consolida los permisos del modelo + * @param main_model Modelo de datos a procesar + * @returns Modelo procesado con permisos consolidados + */ + public async cleanModel(main_model: any): Promise { + // Clonar roles para no modificar original + const roles = _.cloneDeep(main_model.ds.metadata.model_granted_roles); + const model_granted_roles: any[] = []; + const mapRoles = new Map(); - const addOrUpdateRole = (role: any, key: string) => { - const existingRole = mapRoles.get(key); - if (existingRole) { - if (role.type === "users") { - existingRole.users = Array.from(new Set([...existingRole.users, ...role.users])); - existingRole.usersName = Array.from(new Set([...existingRole.usersName, ...role.usersName])); - } else if (role.type === "groups") { - existingRole.groups = Array.from(new Set([...existingRole.groups, ...role.groups])); - existingRole.groupsName = Array.from(new Set([...existingRole.groupsName, ...role.groupsName])); - } - } else { - mapRoles.set(key, _.cloneDeep(role)); - } - }; + /** + * Añade o actualiza un rol en el mapa de roles + * @param role Rol a procesar + * @param key Clave única del rol + */ + const addOrUpdateRole = (role: any, key: string) => { + const existingRole = mapRoles.get(key); + if (existingRole) { + // Fusionar usuarios o grupos según tipo + if (role.type === "users") { + existingRole.users = Array.from(new Set([...existingRole.users, ...role.users])); + existingRole.usersName = Array.from(new Set([...existingRole.usersName, ...role.usersName])); + } else if (role.type === "groups") { + existingRole.groups = Array.from(new Set([...existingRole.groups, ...role.groups])); + existingRole.groupsName = Array.from(new Set([...existingRole.groupsName, ...role.groupsName])); + } + } else { + // Crear nuevo rol si no existe + mapRoles.set(key, _.cloneDeep(role)); + } + }; - roles.forEach((role: any) => { - const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.dynamic}-${role.value?.join(',')}`; - addOrUpdateRole(role, key); - }); + // Procesar cada rol y generar clave única + roles.forEach((role: any) => { + const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.dynamic}-${role.value?.join(',')}`; + addOrUpdateRole(role, key); + }); - mapRoles.forEach((value) => model_granted_roles.push(value)); + // Convertir mapa a array + mapRoles.forEach((value) => model_granted_roles.push(value)); - // Recuperar permisos adicionales de Mongo - const finder = await DataSourceSchema.find({ _id: "111111111111111111111111" }); - const mgsmap = finder - .map((e) => e.ds.metadata.model_granted_roles) - .reduce((acc, val) => acc.concat(val), []); - // Combinar permisos únicos - const filterUniqueRoles = (roles: any[], comparator: (a: any, b: any) => boolean) => { - const seen = new Map(); - return roles.filter((role) => { - const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.users?.join(',')}-${role.groups?.join(',')}`; - if (seen.has(key)) { - return false; - } - seen.set(key, role); - return true; - }); - }; + // Obtener permisos existentes de MongoDB + const finder = await DataSourceSchema.find({ _id: "111111111111111111111111" }); + const mgsmap = finder + .map((e) => e.ds.metadata.model_granted_roles) + .reduce((acc, val) => acc.concat(val), []); - const uniqueRoles = filterUniqueRoles(model_granted_roles, _.isEqual); + /** + * Filtra roles duplicados usando una clave compuesta + * @param roles Array de roles a filtrar + * @param comparator Función de comparación + * @returns Array de roles únicos + */ + const filterUniqueRoles = (roles: any[], comparator: (a: any, b: any) => boolean) => { + const seen = new Map(); + return roles.filter((role) => { + const key = `${role.table}-${role.column}-${role.type}-${role.global}-${role.none}-${role.permission}-${role.users?.join(',')}-${role.groups?.join(',')}`; + if (seen.has(key)) return false; + seen.set(key, role); + return true; + }); + }; - // Marcar origen de permisos - uniqueRoles.forEach((role) => (role.source = "update_model")); + // Obtener roles únicos y marcar origen + const uniqueRoles = filterUniqueRoles(model_granted_roles, _.isEqual); + uniqueRoles.forEach((role) => (role.source = "update_model")); - if (mgsmap.length) { - const userRoles = mgsmap.filter( - (r: any) => r?.source === "SDA" && !r.groupsName.some((name: string) => name.startsWith("SCRM_")) - ); - main_model.ds.metadata.model_granted_roles = [...uniqueRoles, ...userRoles]; - } else { - main_model.ds.metadata.model_granted_roles = uniqueRoles; - } + // Combinar roles según existencia de permisos en MongoDB + if (mgsmap.length) { + // Filtrar roles de usuario SDA no-SCRM + const userRoles = mgsmap.filter( + (r: any) => r?.source === "SDA" && !r.groupsName.some((name: string) => name.startsWith("SCRM_")) + ); + main_model.ds.metadata.model_granted_roles = [...uniqueRoles, ...userRoles]; + } else { + main_model.ds.metadata.model_granted_roles = uniqueRoles; + } - return main_model; - } -} + return main_model; + } +} \ No newline at end of file diff --git a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts index 9afa64533..38fd38583 100644 --- a/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts +++ b/eda/eda_api/lib/module/updateModel/service/usersAndGroupsToMongo.ts @@ -1,3 +1,26 @@ +/** +* Módulo de sincronización de usuarios y permisos entre CRM y MongoDB. +* +* Este módulo implementa la lógica para mantener sincronizados los usuarios y grupos entre +* un CRM basado en MariaDB y una base de datos MongoDB. Las principales funcionalidades son: +* +* - Sincronización de usuarios: importa usuarios del CRM, mantiene contraseñas actualizadas +* y elimina usuarios inactivos preservando cuentas del sistema. +* +* - Gestión de grupos: mantiene grupos SCRM_ sincronizados con el CRM mientras preserva +* grupos propios de SDA (EDA_ADMIN, EDA_RO, etc). +* +* - Relaciones bidireccionales: garantiza consistencia en las relaciones usuario-grupo +* tanto desde la perspectiva del usuario como del grupo. +* +* - Caché optimizada: implementa un sistema de caché que se reconstruye en cada ejecución +* para optimizar consultas a MongoDB durante el proceso. +* +* - Procesamiento por lotes: realiza operaciones de escritura en lotes para mejor rendimiento. +* +* - Tratamiento especial para grupo EDA_ADMIN: mantiene sincronizada la pertenencia a este grupo +* según sda_def_user_groups. +*/ import mongoose, { connections, Model, mongo, Mongoose, QueryOptions } from "mongoose"; import User, { IUser } from "../../admin/users/model/user.model"; import Group, { IGroup } from "../../admin/groups/model/group.model"; diff --git a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts index 1dd031352..71c584f8b 100644 --- a/eda/eda_api/lib/module/updateModel/updateModel.controller.ts +++ b/eda/eda_api/lib/module/updateModel/updateModel.controller.ts @@ -1,3 +1,10 @@ +/** +* Este módulo gestiona la actualización del modelo de datos de SinergiaDA. +* Maneja la sincronización entre la base de datos MariaDB del CRM y MongoDB, +* incluyendo la estructura de tablas, columnas, relaciones, enumeraciones y +* permisos de usuarios/grupos. +*/ + import { kMaxLength } from "buffer"; import { NextFunction, Request, Response } from "express"; import Group, { IGroup } from "../admin/groups/model/group.model"; @@ -7,7 +14,6 @@ import { userAndGroupsToMongo } from "./service/usersAndGroupsToMongo"; import { Enumerations } from "./service/enumerations"; import { pushModelToMongo } from "./service/push.Model.to.Mongo"; import path from 'path'; - import fs from "fs"; import { CleanModel } from "./service/cleanModel"; @@ -15,607 +21,700 @@ const mariadb = require("mariadb"); const sinergiaDatabase = require("../../../config/sinergiacrm.config"); export class updateModel { - /** Updates the SinergiaDA data model of an instance */ - static async update(req: Request, res: Response) { - let crm_to_eda: any = {}; - let modelToExport: any = {}; - let grantedRolesAt: any = []; - let enumerator: any; - let connection: any; - console.time("UpdateModel"); - connection = await mariadb.createConnection(sinergiaDatabase.sinergiaConn); - console.timeLog("UpdateModel", "(Create connection)"); - - try { - /** Checks if columns and tables defined in the model exist */ - await updateModel.checkSinergiaModel(connection); - console.timeLog("UpdateModel", "(Checking model)"); - - // Connect to client database to extract data and create our EDA model - // Select tables - await connection - .query( - " select `table`, label , description, visible from sda_def_tables sdt union all " + - " select distinct master_table , master_table ,master_table , 0 as visible " + - " from sda_def_enumerations sde union all " + - " select distinct bridge_table , bridge_table ,bridge_table , 0 as visible " + - " from sda_def_enumerations sde " + - " where bridge_table != '' " - ) - .then(async (rows, err1) => { - if (err1) { - console.log("Error getting tables list"); - throw err1; - } - let tables = rows; - // Select columns - const my_query = - " select sdc.`table`, sdc.`column`,`type`,sdc.label, sdc.description, sdc.decimals, sdc.aggregations, sdc.visible, sdc.stic_type, sdc.sda_hidden " + - " FROM sda_def_columns sdc " + - " union " + - " select master_table , master_id , 'text', master_id , master_id , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " union " + - " select master_table , master_column , 'text', master_column , master_column , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " union " + - " select bridge_table , source_bridge , 'text', source_bridge , source_bridge , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " where bridge_table != '' " + - " union " + - " select bridge_table , target_bridge , 'text', target_bridge , target_bridge , 0, 'none', 0 , 'varchar', 0 " + - " from sda_def_enumerations sde " + - " where bridge_table != '' "; - await connection.query(my_query).then(async rows => { - let columns = rows; - // Select relationships - await connection - .query( - ` - SELECT distinct source_table, source_column, target_table, target_column, label , 0 as direccion - FROM sda_def_relationships - union - SELECT target_table as source_table, target_column as source_column , - source_table as target_table , source_column as target_column, label as label , 1 as direccion - FROM sda_def_relationships - where source_table != target_table - union - SELECT source_table , source_column , - master_table , master_id as target_column, 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table is null or bridge_table = '' - union - SELECT master_table , master_id , - source_table , source_column , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table is null or bridge_table = '' - union - SELECT source_table , source_bridge as source_column , - bridge_table , source_bridge, 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' - union - SELECT bridge_table , source_bridge, - source_table , source_bridge as source_column , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' - union - SELECT bridge_table , target_bridge, - master_table , master_id , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' - union - SELECT master_table , master_id, - bridge_table , target_bridge , 'xx-bridge|xx-bridge' , 2 as direccion - FROM sda_def_enumerations - where bridge_table != '' ` - ) - .then(async rows => { - let relations = rows; - // Select users (all users, not only active, neccesary to determine users to delete in SDA) - await connection - .query( - "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE password IS NOT NULL ;" - ) - .then(async users => { - let users_crm = users; - // Select EDA roles - await connection - .query( - 'select "EDA_USER_ROLE" as role, b.name, "" as user_name from sda_def_groups b union select "EDA_USER_ROLE" as role, g.name as name , g.user_name from sda_def_user_groups g; ' - ) - .then(async role => { - let roles = role; - await connection - .query( - ' select distinct a.user_name as name, a.`table`, "id" as `column`, a.`group` from sda_def_permissions a where a.`group` != "" ; ' - ) - .then(async granted => { - let fullTablePermissionsForRoles = granted; - // Select enumerations - await connection - .query( - " select source_table , source_column , master_table, master_id, master_column, bridge_table, source_bridge, target_bridge, stic_type, info from sda_def_enumerations sde ;" - ) - .then(async enums => { - let ennumeration = enums; - await connection - .query(" select user_name as name, `table` from sda_def_permissions ") - .then(async permi => { - let fullTablePermissionsForUsers = permi; - - await connection - .query( - " select distinct user_name as name, `table`, 'id' as `column`, group_concat( distinct `group`) as `group` from sda_def_permissions where `group` != '' group by 1,2,3 " - ) - .then(async permiCol => { - let dynamicPermisssionsForGroup = permiCol; - /** Now that we have all data, proceed to build the model */ - const query = - 'select user_name as name, `table` as tabla , `column` as columna from sda_def_permissions where stic_permission_source in ( "ACL_ALLOW_OWNER")'; - await connection.query(query).then(async customUserPermissionsValue => { - console.timeLog("UpdateModel", "(Run MariaDB queries)"); - let dynamicPermisssionsForUser = customUserPermissionsValue; - - try { - crm_to_eda = await userAndGroupsToMongo.crm_to_eda_UsersAndGroups( - users_crm, - roles - ); - console.timeLog("UpdateModel", "(Syncs users and groups)"); - } catch (e) { - console.log("Error 1", e); - res.status(500).json({ status: "ko" }); - } - try { - grantedRolesAt = await updateModel.grantedRolesToModel( - fullTablePermissionsForRoles, - tables, - fullTablePermissionsForUsers, - dynamicPermisssionsForGroup, - dynamicPermisssionsForUser - ); - console.timeLog("UpdateModel", "(Converts CRM roles to EDA)"); - } catch (e) { - console.log("Error 2", e); - res.status(500).json({ status: "ko" }); - } - - try { - modelToExport = updateModel.createModel( - tables, - columns, - relations, - grantedRolesAt, - ennumeration, - res - ); - console.timeLog("UpdateModel", "(Creating Model)"); - } catch (e) { - console.log("Error 3", e); - res.status(500).json({ status: "ko" }); - } - }); - - connection.end(); - }); - }); - }); - }); - }); - }); - }); - }); - }); - } catch (e) { - console.log("Error : ", e); - } - } - /** Checks if columns and tables defined in the model exist */ - static async checkSinergiaModel(con: any) { - let tablas = []; - let columns = []; - let successfulQueries = 0; - - try { - const dataset = await con - .query("select sdc.`table` as tabla, sdc.`column` as `column` FROM sda_def_columns sdc") - .catch(err => { - console.log("Error retrieving tables to check:", err); - throw err; - }); - - tablas = [...new Set(dataset.map(item => item.tabla))]; - - for (const tabla of tablas) { - columns = [...new Set(dataset.map(item => (tabla === item.tabla ? item.column : null)))].filter( - item => item != null - ); - - const sql = " select " + columns.toString() + " from " + tabla + " limit 1 \n"; - let nexSql = sql.replace("select ,", "select ").replace(", from", " from "); - - try { - await con.query(nexSql); - successfulQueries++; - } catch (err) { - console.log(`Error executing query for table ${tabla}:`, err); - } - } - } catch (err) { - console.log("Error in model check process:", err); - } - } - - /** Generates and processes model roles */ - static async grantedRolesToModel( - fullTablePermissionsForRoles: any, - crmTables: any, - fullTablePermissionsForUsers: any, - dynamicPermisssionsForGroup: any, - dynamicPermisssionsForUser: any -) { - const destGrantedRoles = []; - let gr, gr2, gr3, gr4, gr5 = {}; - - const usersFound = await User.find(); - const mongoGroups = await Group.find(); - - // Función auxiliar para verificar duplicados - const hasExistingFullTablePermission = (newRole: any) => { - return destGrantedRoles.some(existing => - existing.column === "fullTable" && - existing.table === newRole.table && - existing.type === newRole.type && - existing.users?.toString() === newRole.users?.toString() - ); - }; - - fullTablePermissionsForRoles.forEach(line => { - let match = mongoGroups.filter(i => { - return i.name === line.group; - }); - let mongoId: String; - let mongoGroup: String; - if (match.length == 1 && line.group !== undefined) { - mongoId = match[0]._id.toString(); - mongoGroup = match[0].name.toString(); - if (line.name != null) { - // Process group converted to user - const found = usersFound.find(i => i.email == line.name); - gr = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "users" - }; - - // Verificar duplicados antes de agregar - if (!hasExistingFullTablePermission(gr)) { - destGrantedRoles.push(gr); - } - } else { - gr = { - groups: [mongoId], - groupsName: [mongoGroup], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "groups" - }; - destGrantedRoles.push(gr); - } - } - }); - - fullTablePermissionsForUsers.forEach(line => { - const found = usersFound.find(i => i.email == line.name); - if (found) { - gr3 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "fullTable", - global: true, - permission: true, - type: "users" - }; - if (!hasExistingFullTablePermission(gr3)) { - destGrantedRoles.push(gr3); - } - } - }); - - dynamicPermisssionsForGroup.forEach(line => { - const match = mongoGroups.filter(i => { - return i.name === line.group.split(",")[0]; - }); - let mongoId: String; - if (match.length == 1 && line.group !== undefined) { - mongoId = match[0]._id.toString(); - let group_name: String = " '" + line.group + "' "; - let table_name: String = " '" + line.table + "' "; - let valueAt: String = - " select record_id from sda_def_security_group_records" + - " where `group` in ( " + - group_name.split(",").join("','") + - ") and `table` = " + - table_name; - if (line.name != null) { - // Process group converted to user - const found = usersFound.find(i => i.email == line.name); - gr4 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: line.column, - dynamic: true, - global: false, - type: "users", - value: [valueAt] - }; - destGrantedRoles.push(gr4); - - let valueAt2: String = " select `id` from " + line.table + " where `assigned_user_name` = 'EDA_USER' "; - gr5 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.table, - column: "id", - global: false, - permission: true, - dynamic: true, - type: "users", - value: [valueAt2] - }; - destGrantedRoles.push(gr5); - } - } - }); - - dynamicPermisssionsForUser.forEach(line => { - const found = usersFound.find(i => i.email == line.name); - if (found) { - let valueAt: String = - "select `" + line.columna + "` from " + line.tabla + " where `" + line.columna + "` = 'EDA_USER' "; - gr5 = { - users: [found._id], - usersName: [line.name], - none: false, - table: line.tabla, - column: line.columna, - global: false, - permission: true, - dynamic: true, - type: "users", - value: [valueAt] - }; - destGrantedRoles.push(gr5); - } - }); - - return destGrantedRoles; -} - - static createModel( - tables: any, - columns: any, - relations: any, - grantedRoles: any, - ennumeration: any, - res: any - ): string[] { - let visible = false; - - /** Process and generate tables structure */ - const destTables = []; - for (let i = 0; i < tables.length; i++) { - if (tables[i].visible == 1) { - visible = true; - } else { - visible = false; - } - - var tabla = { - table_name: tables[i].table, - columns: [], - relations: [], - display_name: { - default: tables[i].label, - localized: [] - }, - description: { - default: tables[i].description, - localized: [] - }, - visible: visible, - table_granted_roles: [], - table_type: [], - tableCount: 0, - no_relations: [] - }; - - const destColumns: any[] = updateModel.getColumnsForTable(tables[i].table, columns, ennumeration); - tabla.columns = destColumns; - - const destRelations: any[] = updateModel.getRelations(tables[i].table, relations); - tabla.relations = destRelations; - - destTables.push(tabla); - } - - this.extractJsonModelAndPushToMongo(destTables, grantedRoles, res); - - return destTables; - } - static getAggretations(aggregations: string) { - const aggArray = aggregations.split(","); - const agg = []; - if (aggArray.indexOf("sum") >= 0) { - agg.push({ value: "sum", display_name: "Sum" }); - } - if (aggArray.indexOf("avg") >= 0) { - agg.push({ value: "avg", display_name: "Average" }); - } - if (aggArray.indexOf("max") >= 0) { - agg.push({ value: "max", display_name: "Maximum" }); - } - if (aggArray.indexOf("min") >= 0) { - agg.push({ value: "min", display_name: "Minimum" }); - } - if (aggArray.indexOf("count") >= 0) { - agg.push({ value: "count", display_name: "Count Values" }); - } - if (aggArray.indexOf("count_distinct") > 0) { - agg.push({ value: "count_distinct", display_name: "Distinct Values" }); - } - agg.push({ value: "none", display_name: "None" }); - return agg; - } - - /** Retrieves and processes columns for a specific table */ - static getColumnsForTable(table: string, columns: any, ennumeration: any) { - const destColumns = []; - - const agg_none = [ - { - value: "none", - display_name: "none" - } - ]; - - let agg_used = {}; - let colVisible = false; - - for (let i = 0; i < columns.length; i++) { - let c = columns[i]; - - // Determine aggregators based on column types - if (columns[i].type) { - agg_used = this.getAggretations(columns[i].aggregations); - } else { - agg_used = agg_none; - } - - if (columns[i].visible == 1) { - colVisible = true; - } else { - colVisible = false; - } - - if (c.table == table) { - c = { - column_name: columns[i].column, - column_type: columns[i].type == "enumeration" ? "text" : columns[i].type, - display_name: { - default: columns[i].label, - localized: [] - }, - description: { - default: columns[i].description, - localized: [] - }, - aggregation_type: agg_used, - visible: colVisible, - minimumFractionDigits: columns[i].decimals, - column_granted_roles: [], - row_granted_roles: [], - tableCount: 0, - valueListSource: {}, - hidden: columns[i].sda_hidden - }; - - // Process valueListSource for each column based on source_table - const foundTable = ennumeration.filter(j => j.source_table == table); - foundTable.forEach(u => { - if (u.source_column == c.column_name) { - c.valueListSource = Enumerations.enumeration_to_column(u); + + /** + * Actualiza el modelo de datos de una instancia de SinergiaDA + * Extrae datos del CRM, procesa la estructura y sincroniza con MongoDB + * @param req Petición HTTP + * @param res Respuesta HTTP + */ + static async update(req: Request, res: Response) { + // Variables para almacenar datos durante el proceso + let crm_to_eda: any = {}; + let modelToExport: any = {}; + let grantedRolesAt: any = []; + let enumerator: any; + let connection: any; + + // Inicio del proceso y conexión a BD + console.time("UpdateModel"); + connection = await mariadb.createConnection(sinergiaDatabase.sinergiaConn); + console.timeLog("UpdateModel", "(Create connection)"); + + try { + // Validación inicial del modelo + await updateModel.checkSinergiaModel(connection); + console.timeLog("UpdateModel", "(Checking model)"); + + await connection + .query( + // Query que une tablas base, maestras y puente + " select `table`, label , description, visible from sda_def_tables sdt union all " + + " select distinct master_table , master_table ,master_table , 0 as visible " + + " from sda_def_enumerations sde union all " + + " select distinct bridge_table , bridge_table ,bridge_table , 0 as visible " + + " from sda_def_enumerations sde " + + " where bridge_table != '' " + ) + .then(async (rows, err1) => { + if (err1) { + console.log("Error getting tables list"); + throw err1; + } + let tables = rows; + + // Query para obtener definición de columnas + const my_query = + " select sdc.`table`, sdc.`column`,`type`,sdc.label, sdc.description, sdc.decimals, sdc.aggregations, sdc.visible, sdc.stic_type, sdc.sda_hidden " + + " FROM sda_def_columns sdc " + + " union " + + " select master_table , master_id , 'text', master_id , master_id , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " union " + + " select master_table , master_column , 'text', master_column , master_column , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " union " + + " select bridge_table , source_bridge , 'text', source_bridge , source_bridge , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " where bridge_table != '' " + + " union " + + " select bridge_table , target_bridge , 'text', target_bridge , target_bridge , 0, 'none', 0 , 'varchar', 0 " + + " from sda_def_enumerations sde " + + " where bridge_table != '' "; + await connection.query(my_query).then(async rows => { + let columns = rows; + + // Consulta para obtener relaciones entre tablas + await connection + .query( + ` + SELECT distinct source_table, source_column, target_table, target_column, label , 0 as direccion + FROM sda_def_relationships + union + SELECT target_table as source_table, target_column as source_column , + source_table as target_table , source_column as target_column, label as label , 1 as direccion + FROM sda_def_relationships + where source_table != target_table + union + SELECT source_table , source_column , + master_table , master_id as target_column, 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table is null or bridge_table = '' + union + SELECT master_table , master_id , + source_table , source_column , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table is null or bridge_table = '' + union + SELECT source_table , source_bridge as source_column , + bridge_table , source_bridge, 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' + union + SELECT bridge_table , source_bridge, + source_table , source_bridge as source_column , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' + union + SELECT bridge_table , target_bridge, + master_table , master_id , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' + union + SELECT master_table , master_id, + bridge_table , target_bridge , 'xx-bridge|xx-bridge' , 2 as direccion + FROM sda_def_enumerations + where bridge_table != '' ` + ) + .then(async rows => { + let relations = rows; + + // Obtener usuarios (activos e inactivos) + await connection + .query( + "SELECT name as name, user_name as email, password as password, active as active FROM sda_def_users WHERE password IS NOT NULL ;" + ) + .then(async users => { + let users_crm = users; + + // Obtener roles EDA + await connection + .query( + 'select "EDA_USER_ROLE" as role, b.name, "" as user_name from sda_def_groups b union select "EDA_USER_ROLE" as role, g.name as name , g.user_name from sda_def_user_groups g; ' + ) + .then(async role => { + let roles = role; + + // Obtener permisos de grupos sobre tablas + await connection + .query( + ' select distinct a.user_name as name, a.`table`, "id" as `column`, a.`group` from sda_def_permissions a where a.`group` != "" ; ' + ) + .then(async granted => { + let fullTablePermissionsForRoles = granted; + + // Obtener enumeraciones + await connection + .query( + " select source_table , source_column , master_table, master_id, master_column, bridge_table, source_bridge, target_bridge, stic_type, info from sda_def_enumerations sde ;" + ) + .then(async enums => { + let ennumeration = enums; + + // Obtener permisos de usuarios sobre tablas + await connection + .query(" select user_name as name, `table` from sda_def_permissions ") + .then(async permi => { + let fullTablePermissionsForUsers = permi; + + await connection + .query( + " select distinct user_name as name, `table`, 'id' as `column`, group_concat( distinct `group`) as `group` from sda_def_permissions where `group` != '' group by 1,2,3 " + ) + .then(async permiCol => { + let dynamicPermisssionsForGroup = permiCol; + + // Obtener permisos dinámicos de usuarios + const query = + 'select user_name as name, `table` as tabla , `column` as columna from sda_def_permissions where stic_permission_source in ( "ACL_ALLOW_OWNER")'; + await connection.query(query).then(async customUserPermissionsValue => { + console.timeLog("UpdateModel", "(Run MariaDB queries)"); + let dynamicPermisssionsForUser = customUserPermissionsValue; + + try { + // Sincronizar usuarios y grupos + crm_to_eda = await userAndGroupsToMongo.crm_to_eda_UsersAndGroups( + users_crm, + roles + ); + console.timeLog("UpdateModel", "(Syncs users and groups)"); + } catch (e) { + console.log("Error 1", e); + res.status(500).json({ status: "ko" }); + } + + try { + // Convertir permisos CRM a modelo EDA + grantedRolesAt = await updateModel.grantedRolesToModel( + fullTablePermissionsForRoles, + tables, + fullTablePermissionsForUsers, + dynamicPermisssionsForGroup, + dynamicPermisssionsForUser + ); + console.timeLog("UpdateModel", "(Converts CRM roles to EDA)"); + } catch (e) { + console.log("Error 2", e); + res.status(500).json({ status: "ko" }); + } + + try { + // Crear modelo final + modelToExport = updateModel.createModel( + tables, + columns, + relations, + grantedRolesAt, + ennumeration, + res + ); + console.timeLog("UpdateModel", "(Creating Model)"); + } catch (e) { + console.log("Error 3", e); + res.status(500).json({ status: "ko" }); + } + }); + + connection.end(); + }); + }); + }); + }); + }); + }); + }); + }); + }); + } catch (e) { + console.log("Error : ", e); + } + } + + /** + * Verifica la existencia de columnas y tablas definidas en el modelo + * @param con Conexión a la base de datos + */ + static async checkSinergiaModel(con: any) { + // Arrays para almacenar metadatos + let tablas = []; + let columns = []; + let successfulQueries = 0; + + try { + // Obtener listado de tablas y columnas + const dataset = await con + .query("select sdc.`table` as tabla, sdc.`column` as `column` FROM sda_def_columns sdc") + .catch(err => { + console.log("Error retrieving tables to check:", err); + throw err; + }); + + // Extraer nombres únicos de tablas + tablas = [...new Set(dataset.map(item => item.tabla))]; + + // Verificar cada tabla y sus columnas + for (const tabla of tablas) { + columns = [...new Set(dataset.map(item => (tabla === item.tabla ? item.column : null)))].filter( + item => item != null + ); + + // Construir y ejecutar query de prueba + const sql = " select " + columns.toString() + " from " + tabla + " limit 1 \n"; + let nexSql = sql.replace("select ,", "select ").replace(", from", " from "); + + try { + await con.query(nexSql); + successfulQueries++; + } catch (err) { + console.log(`Error executing query for table ${tabla}:`, err); + } + } + } catch (err) { + console.log("Error in model check process:", err); + } + } + + /** + * Genera y procesa los roles del modelo EDA a partir de los permisos del CRM + * @param fullTablePermissionsForRoles Permisos de grupos sobre tablas completas + * @param crmTables Tablas del CRM + * @param fullTablePermissionsForUsers Permisos de usuarios sobre tablas completas + * @param dynamicPermisssionsForGroup Permisos dinámicos para grupos + * @param dynamicPermisssionsForUser Permisos dinámicos para usuarios + * @returns Array de roles procesados + */ + static async grantedRolesToModel( + fullTablePermissionsForRoles: any, + crmTables: any, + fullTablePermissionsForUsers: any, + dynamicPermisssionsForGroup: any, + dynamicPermisssionsForUser: any + ) { + const destGrantedRoles = []; + let gr, gr2, gr3, gr4, gr5 = {}; + + // Obtener usuarios y grupos de MongoDB + const usersFound = await User.find(); + const mongoGroups = await Group.find(); + + // Función para verificar permisos duplicados + const hasExistingFullTablePermission = (newRole: any) => { + return destGrantedRoles.some(existing => + existing.column === "fullTable" && + existing.table === newRole.table && + existing.type === newRole.type && + existing.users?.toString() === newRole.users?.toString() + ); + }; + + // Procesar permisos de grupos sobre tablas + fullTablePermissionsForRoles.forEach(line => { + let match = mongoGroups.filter(i => { + return i.name === line.group; + }); + let mongoId: String; + let mongoGroup: String; + if (match.length == 1 && line.group !== undefined) { + mongoId = match[0]._id.toString(); + mongoGroup = match[0].name.toString(); + if (line.name != null) { + // Procesar grupo convertido a usuario + const found = usersFound.find(i => i.email == line.name); + gr = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "users" + }; + + // Verificar duplicados antes de agregar + if (!hasExistingFullTablePermission(gr)) { + destGrantedRoles.push(gr); + } + } else { + gr = { + groups: [mongoId], + groupsName: [mongoGroup], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "groups" + }; + destGrantedRoles.push(gr); + } + } + }); + + // Procesar permisos de usuarios sobre tablas + fullTablePermissionsForUsers.forEach(line => { + const found = usersFound.find(i => i.email == line.name); + if (found) { + gr3 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "fullTable", + global: true, + permission: true, + type: "users" + }; + if (!hasExistingFullTablePermission(gr3)) { + destGrantedRoles.push(gr3); + } + } + }); + + // Procesar permisos dinámicos de grupos + dynamicPermisssionsForGroup.forEach(line => { + const match = mongoGroups.filter(i => { + return i.name === line.group.split(",")[0]; + }); + let mongoId: String; + if (match.length == 1 && line.group !== undefined) { + mongoId = match[0]._id.toString(); + let group_name: String = " '" + line.group + "' "; + let table_name: String = " '" + line.table + "' "; + let valueAt: String = + " select record_id from sda_def_security_group_records" + + " where `group` in ( " + + group_name.split(",").join("','") + + ") and `table` = " + + table_name; + if (line.name != null) { + // Procesar grupo convertido a usuario + const found = usersFound.find(i => i.email == line.name); + gr4 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: line.column, + dynamic: true, + global: false, + type: "users", + value: [valueAt] + }; + destGrantedRoles.push(gr4); + + let valueAt2: String = " select `id` from " + line.table + " where `assigned_user_name` = 'EDA_USER' "; + gr5 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.table, + column: "id", + global: false, + permission: true, + dynamic: true, + type: "users", + value: [valueAt2] + }; + destGrantedRoles.push(gr5); + } + } + }); + + // Procesar permisos dinámicos de usuarios + dynamicPermisssionsForUser.forEach(line => { + const found = usersFound.find(i => i.email == line.name); + if (found) { + let valueAt: String = + "select `" + line.columna + "` from " + line.tabla + " where `" + line.columna + "` = 'EDA_USER' "; + gr5 = { + users: [found._id], + usersName: [line.name], + none: false, + table: line.tabla, + column: line.columna, + global: false, + permission: true, + dynamic: true, + type: "users", + value: [valueAt] + }; + destGrantedRoles.push(gr5); + } + }); + + return destGrantedRoles; + } + + /** + * Crea el modelo de datos completo con tablas, columnas y relaciones + * @param tables Tablas del modelo + * @param columns Columnas de las tablas + * @param relations Relaciones entre tablas + * @param grantedRoles Roles y permisos + * @param ennumeration Enumeraciones del modelo + * @param res Respuesta HTTP + * @returns Array de tablas procesadas + */ + static createModel( + tables: any, + columns: any, + relations: any, + grantedRoles: any, + ennumeration: any, + res: any + ): string[] { + let visible = false; + + // Procesar estructura de tablas + const destTables = []; + for (let i = 0; i < tables.length; i++) { + // Determinar visibilidad + if (tables[i].visible == 1) { + visible = true; + } else { + visible = false; + } + + // Crear estructura base de tabla + var tabla = { + table_name: tables[i].table, + columns: [], + relations: [], + display_name: { + default: tables[i].label, + localized: [] + }, + description: { + default: tables[i].description, + localized: [] + }, + visible: visible, + table_granted_roles: [], + table_type: [], + tableCount: 0, + no_relations: [] + }; + + // Procesar columnas y relaciones + const destColumns: any[] = updateModel.getColumnsForTable(tables[i].table, columns, ennumeration); + tabla.columns = destColumns; + + const destRelations: any[] = updateModel.getRelations(tables[i].table, relations); + tabla.relations = destRelations; + + destTables.push(tabla); + } + + // Enviar modelo a MongoDB + this.extractJsonModelAndPushToMongo(destTables, grantedRoles, res); + + return destTables; + } + + /** + * Genera la configuración de agregaciones para las columnas + * @param aggregations String con tipos de agregación separados por comas + * @returns Array de configuraciones de agregación + */ + static getAggretations(aggregations: string) { + // Procesar tipos de agregación + const aggArray = aggregations.split(","); + const agg = []; + + // Añadir configuraciones según tipos + if (aggArray.indexOf("sum") >= 0) { + agg.push({ value: "sum", display_name: "Sum" }); + } + if (aggArray.indexOf("avg") >= 0) { + agg.push({ value: "avg", display_name: "Average" }); + } + if (aggArray.indexOf("max") >= 0) { + agg.push({ value: "max", display_name: "Maximum" }); + } + if (aggArray.indexOf("min") >= 0) { + agg.push({ value: "min", display_name: "Minimum" }); + } + if (aggArray.indexOf("count") >= 0) { + agg.push({ value: "count", display_name: "Count Values" }); + } + if (aggArray.indexOf("count_distinct") > 0) { + agg.push({ value: "count_distinct", display_name: "Distinct Values" }); + } + agg.push({ value: "none", display_name: "None" }); + return agg; + } + + /** + * Procesa y devuelve las columnas para una tabla específica + * @param table Nombre de la tabla + * @param columns Array de columnas a procesar + * @param ennumeration Configuración de enumeraciones + * @returns Array de columnas procesadas + */ + static getColumnsForTable(table: string, columns: any, ennumeration: any) { + const destColumns = []; + + // Configuración por defecto de agregación + const agg_none = [ + { + value: "none", + display_name: "none" + } + ]; + + let agg_used = {}; + let colVisible = false; + + // Procesar cada columna + for (let i = 0; i < columns.length; i++) { + let c = columns[i]; + + // Determinar agregadores según tipo + if (columns[i].type) { + agg_used = this.getAggretations(columns[i].aggregations); + } else { + agg_used = agg_none; + } + + // Determinar visibilidad + if (columns[i].visible == 1) { + colVisible = true; + } else { + colVisible = false; + } + + // Procesar columna si pertenece a la tabla + if (c.table == table) { + c = { + column_name: columns[i].column, + column_type: columns[i].type == "enumeration" ? "text" : columns[i].type, + display_name: { + default: columns[i].label, + localized: [] + }, + description: { + default: columns[i].description, + localized: [] + }, + aggregation_type: agg_used, + visible: colVisible, + minimumFractionDigits: columns[i].decimals, + column_granted_roles: [], + row_granted_roles: [], + tableCount: 0, + valueListSource: {}, + hidden: columns[i].sda_hidden + }; + + // Procesar fuente de valores si existe + const foundTable = ennumeration.filter(j => j.source_table == table); + foundTable.forEach(u => { + if (u.source_column == c.column_name) { + c.valueListSource = Enumerations.enumeration_to_column(u); + } + }); + destColumns.push(c); + } + } + return destColumns; + } + + /** + * Procesa y devuelve las relaciones para una tabla específica + * @param table Nombre de la tabla + * @param relations Array de relaciones a procesar + * @returns Array de relaciones procesadas + */ + static getRelations(table: string, relations: any) { + const destRelations = []; + + // Procesar cada relación + for (let i = 0; i < relations.length; i++) { + let r = relations[i]; + if (r.source_table == table) { + let rr = { + source_table: relations[i].source_table, + source_column: [relations[i].source_column], + target_table: relations[i].target_table, + target_column: [relations[i].target_column], + visible: true, + bridge: relations[i].label == "xx-bridge|xx-bridge" ? true : false, + display_name: { + default: relations[i].direccion === 0 ? relations[i].label.split("|")[0] : relations[i].label.split("|")[1], + localized: [] + }, + autorelation: relations[i].source_table === relations[i].target_table ? true : false + }; + destRelations.push(rr); + } + } + + return destRelations; + } + + /** + * Formatea y envía el modelo final a MongoDB + * @param tables Tablas procesadas + * @param grantedRoles Roles y permisos + * @param res Respuesta HTTP + */ + static async extractJsonModelAndPushToMongo(tables: any, grantedRoles: any, res: any) { + // Inicio del formateo JSON + console.timeLog("UpdateModel", "(Start JSON formatting)"); + + // Cargar y configurar modelo base + let main_model = await JSON.parse( + fs.readFileSync( + path.join(__dirname, '../../../config/base_datamodel.json'), + "utf-8" + ) + ); + + // Configurar conexión + main_model.ds.connection.host = sinergiaDatabase.sinergiaConn.host; + main_model.ds.connection.database = sinergiaDatabase.sinergiaConn.database; + main_model.ds.connection.port = sinergiaDatabase.sinergiaConn.port; + main_model.ds.connection.user = sinergiaDatabase.sinergiaConn.user; + main_model.ds.connection.poolLimit = sinergiaDatabase.sinergiaConn.connectionLimit; + main_model.ds.connection.password = EnCrypterService.encrypt(sinergiaDatabase.sinergiaConn.password); + + // Asignar datos del modelo + main_model.ds.model.tables = tables; + main_model.ds.metadata.model_granted_roles = await grantedRoles; + + console.timeLog("UpdateModel", "(Model configuration completed)"); + + try { + // Limpiar modelo + const cleanM = new CleanModel(); + main_model = await cleanM.cleanModel(main_model); + console.timeLog("UpdateModel", "(Model cleaning completed)"); + + // Guardar metadata + fs.writeFile(`metadata.json`, JSON.stringify(main_model), { encoding: `utf-8` }, err => { + if (err) { + throw err; + } + }); + console.timeLog("UpdateModel", "(Metadata file written)"); + + // Enviar a MongoDB + await new pushModelToMongo().pushModel(main_model, res); + res.status(200).json({ status: "ok" }); + } catch (e) { + console.log("Error :", e); + res.status(500).json({ status: "ko" }); } - }); - destColumns.push(c); - } - } - return destColumns; - } - - /** Generates and processes relations for a specific table */ - static getRelations(table: string, relations: any) { - const destRelations = []; - - for (let i = 0; i < relations.length; i++) { - let r = relations[i]; - if (r.source_table == table) { - let rr = { - source_table: relations[i].source_table, - source_column: [relations[i].source_column], - target_table: relations[i].target_table, - target_column: [relations[i].target_column], - visible: true, - bridge: relations[i].label == "xx-bridge|xx-bridge" ? true : false, - display_name: { - default: relations[i].direccion === 0 ? relations[i].label.split("|")[0] : relations[i].label.split("|")[1], - localized: [] - }, - autorelation: relations[i].source_table === relations[i].target_table ? true : false - }; - destRelations.push(rr); - } - } - - return destRelations; - } - - /** Formats and pushes the final model to MongoDB */ - static async extractJsonModelAndPushToMongo(tables: any, grantedRoles: any, res: any) { - // Format tables as JSON - console.timeLog("UpdateModel", "(Start JSON formatting)"); // AÑADIR AQUÍ - Línea antes de let main_model - - let main_model = await JSON.parse( - fs.readFileSync( - path.join(__dirname, '../../../config/base_datamodel.json'), - "utf-8" - ) - ); - main_model.ds.connection.host = sinergiaDatabase.sinergiaConn.host; - main_model.ds.connection.database = sinergiaDatabase.sinergiaConn.database; - main_model.ds.connection.port = sinergiaDatabase.sinergiaConn.port; - main_model.ds.connection.user = sinergiaDatabase.sinergiaConn.user; - main_model.ds.connection.poolLimit = sinergiaDatabase.sinergiaConn.connectionLimit; - main_model.ds.connection.password = EnCrypterService.encrypt(sinergiaDatabase.sinergiaConn.password); - main_model.ds.model.tables = tables; - main_model.ds.metadata.model_granted_roles = await grantedRoles; - - console.timeLog("UpdateModel", "(Model configuration completed)"); // AÑADIR AQUÍ - Después de asignar todos los valores - - try { - const cleanM = new CleanModel(); - - main_model = await cleanM.cleanModel(main_model); - console.timeLog("UpdateModel", "(Model cleaning completed)"); // AÑADIR AQUÍ - Después de cleanModel - - fs.writeFile(`metadata.json`, JSON.stringify(main_model), { encoding: `utf-8` }, err => { - if (err) { - throw err; - } - }); - console.timeLog("UpdateModel", "(Metadata file written)"); // AÑADIR AQUÍ - Después de writeFile - - await new pushModelToMongo().pushModel(main_model, res); - // No es necesario añadir timeLog aquí ya que pushModel ya tiene su propio console.timeEnd - - res.status(200).json({ status: "ok" }); - } catch (e) { - console.log("Error :", e); - res.status(500).json({ status: "ko" }); - } -} -} + } +} \ No newline at end of file From 90e4036d7dbe5cc2709413df77d6a1ee9208ba25 Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 10 Dec 2024 16:37:34 +0100 Subject: [PATCH 22/23] remove package.json change --- eda/eda_api/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eda/eda_api/package.json b/eda/eda_api/package.json index ef8686d27..68d63f940 100644 --- a/eda/eda_api/package.json +++ b/eda/eda_api/package.json @@ -5,8 +5,7 @@ "main": "index.js", "scripts": { "start": "nodemon", - "start:forever": "npm run build && forever start -c node ./dist/server.js", - "start:debug": "npm run build && forever start -c node --inspect=0.0.0.0:9229 ./dist/server.js", + "start:forever": "npm run build && forever start -c node ./dist/server.js", "start:pm2": "npm run build && pm2 start ./dist/server.js ", "edapi": "forever start -c node ./dist/server.js", "build": "tsc && npm run movejpgs", From ec84e3752139efc0b8c8cb5ca354ad5da34a0f6e Mon Sep 17 00:00:00 2001 From: juanSTIC Date: Tue, 10 Dec 2024 16:39:09 +0100 Subject: [PATCH 23/23] space --- eda/eda_api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eda/eda_api/package.json b/eda/eda_api/package.json index 68d63f940..6329895a8 100644 --- a/eda/eda_api/package.json +++ b/eda/eda_api/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "start": "nodemon", - "start:forever": "npm run build && forever start -c node ./dist/server.js", + "start:forever": "npm run build && forever start -c node ./dist/server.js", "start:pm2": "npm run build && pm2 start ./dist/server.js ", "edapi": "forever start -c node ./dist/server.js", "build": "tsc && npm run movejpgs",