From d7638a593b0f5981a93abd2319e5bf7b3e79b7d7 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 16 Apr 2020 21:49:31 -0300 Subject: [PATCH 01/40] Import push library --- app/api/server/v1/push.js | 4 +- app/push/server/apn.js | 170 +++++++++++++ app/push/server/gcm.js | 140 +++++++++++ app/push/server/logger.js | 4 + app/push/server/notifications.js | 109 ++++++++ app/push/server/push.api.js | 318 ++++++++++++++++++++++++ app/push/server/push.js | 7 + app/push/server/server.js | 149 +++++++++++ app/statistics/server/lib/statistics.js | 4 +- package-lock.json | 37 +++ package.json | 2 + server/lib/pushConfig.js | 7 +- server/startup/migrations/v181.js | 5 +- 13 files changed, 946 insertions(+), 10 deletions(-) create mode 100644 app/push/server/apn.js create mode 100644 app/push/server/gcm.js create mode 100644 app/push/server/logger.js create mode 100644 app/push/server/notifications.js create mode 100644 app/push/server/push.api.js create mode 100644 app/push/server/push.js create mode 100644 app/push/server/server.js diff --git a/app/api/server/v1/push.js b/app/api/server/v1/push.js index 6c1c9959011a9..1d84d57aa5b02 100644 --- a/app/api/server/v1/push.js +++ b/app/api/server/v1/push.js @@ -1,7 +1,7 @@ import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; -import { Push } from 'meteor/rocketchat:push'; +import { appTokensCollection } from '../../../push/server/push'; import { API } from '../api'; API.v1.addRoute('push.token', { authRequired: true }, { @@ -47,7 +47,7 @@ API.v1.addRoute('push.token', { authRequired: true }, { throw new Meteor.Error('error-token-param-not-valid', 'The required "token" body param is missing or invalid.'); } - const affectedRecords = Push.appCollection.remove({ + const affectedRecords = appTokensCollection.remove({ $or: [{ 'token.apn': token, }, { diff --git a/app/push/server/apn.js b/app/push/server/apn.js new file mode 100644 index 0000000000000..dbb8858260366 --- /dev/null +++ b/app/push/server/apn.js @@ -0,0 +1,170 @@ +import { Meteor } from 'meteor/meteor'; +import { Match } from 'meteor/check'; +import { EJSON } from 'meteor/ejson'; +import _ from 'underscore'; +import apn from 'apn'; + +import { logger } from './logger'; + +let apnConnection; + +export const sendAPN = function(userToken, notification) { + if (Match.test(notification.apn, Object)) { + notification = _.extend({}, notification, notification.apn); + } + + // console.log('sendAPN', notification.from, userToken, notification.title, notification.text, + // notification.badge, notification.priority); + const priority = notification.priority || notification.priority === 0 ? notification.priority : 10; + + const myDevice = new apn.Device(userToken); + + const note = new apn.Notification(); + + note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now. + if (typeof notification.badge !== 'undefined') { + note.badge = notification.badge; + } + if (typeof notification.sound !== 'undefined') { + note.sound = notification.sound; + } + // console.log(notification.contentAvailable); + // console.log("lala2"); + // console.log(notification); + if (typeof notification.contentAvailable !== 'undefined') { + // console.log("lala"); + note.setContentAvailable(notification.contentAvailable); + // console.log(note); + } + + // adds category support for iOS8 custom actions as described here: + // https://developer.apple.com/library/ios/documentation/NetworkingInternet/Conceptual/ + // RemoteNotificationsPG/Chapters/IPhoneOSClientImp.html#//apple_ref/doc/uid/TP40008194-CH103-SW36 + if (typeof notification.category !== 'undefined') { + note.category = notification.category; + } + + note.alert = { + body: notification.text, + }; + + if (typeof notification.title !== 'undefined') { + note.alert.title = notification.title; + } + + // Allow the user to set payload data + note.payload = notification.payload ? { ejson: EJSON.stringify(notification.payload) } : {}; + + note.payload.messageFrom = notification.from; + note.priority = priority; + + + // Store the token on the note so we can reference it if there was an error + note.token = userToken; + + // console.log('I:Send message to: ' + userToken + ' count=' + count); + + apnConnection.pushNotification(note, myDevice); +}; + +// Init feedback from apn server +// This will help keep the appCollection up-to-date, it will help update +// and remove token from appCollection. +export const initFeedback = function({ options, _removeToken }) { + // console.log('Init feedback'); + const feedbackOptions = { + batchFeedback: true, + + // Time in SECONDS + interval: 5, + production: !options.apn.development, + cert: options.certData, + key: options.keyData, + passphrase: options.passphrase, + }; + + const feedback = new apn.Feedback(feedbackOptions); + feedback.on('feedback', function(devices) { + devices.forEach(function(item) { + // Do something with item.device and item.time; + // console.log('A:PUSH FEEDBACK ' + item.device + ' - ' + item.time); + // The app is most likely removed from the device, we should + // remove the token + _removeToken({ + apn: item.device, + }); + }); + }); + + feedback.start(); +}; + +export const initAPN = ({ options, _removeToken }) => { + logger.debug('Push: APN configured'); + + // Allow production to be a general option for push notifications + if (options.production === Boolean(options.production)) { + options.apn.production = options.production; + } + + // Give the user warnings about development settings + if (options.apn.development) { + // This flag is normally set by the configuration file + console.warn('WARNING: Push APN is using development key and certificate'); + } else if (options.apn.gateway) { + // We check the apn gateway i the options, we could risk shipping + // server into production while using the production configuration. + // On the other hand we could be in development but using the production + // configuration. And finally we could have configured an unknown apn + // gateway (this could change in the future - but a warning about typos + // can save hours of debugging) + // + // Warn about gateway configurations - it's more a guide + + if (options.apn.gateway === 'gateway.sandbox.push.apple.com') { + // Using the development sandbox + console.warn('WARNING: Push APN is in development mode'); + } else if (options.apn.gateway === 'gateway.push.apple.com') { + // In production - but warn if we are running on localhost + if (/http:\/\/localhost/.test(Meteor.absoluteUrl())) { + console.warn('WARNING: Push APN is configured to production mode - but server is running from localhost'); + } + } else { + // Warn about gateways we dont know about + console.warn(`WARNING: Push APN unkown gateway "${ options.apn.gateway }"`); + } + } else if (options.apn.production) { + if (/http:\/\/localhost/.test(Meteor.absoluteUrl())) { + console.warn('WARNING: Push APN is configured to production mode - but server is running from localhost'); + } + } else { + console.warn('WARNING: Push APN is in development mode'); + } + + // Check certificate data + if (!options.apn.certData || !options.apn.certData.length) { + console.error('ERROR: Push server could not find certData'); + } + + // Check key data + if (!options.apn.keyData || !options.apn.keyData.length) { + console.error('ERROR: Push server could not find keyData'); + } + + // Rig apn connection + apnConnection = new apn.Connection(options.apn); + + // Listen to transmission errors - should handle the same way as feedback. + apnConnection.on('transmissionError', Meteor.bindEnvironment(function(errCode, notification/* , recipient*/) { + logger.debug('Got error code %d for token %s', errCode, notification.token); + + if ([2, 5, 8].indexOf(errCode) >= 0) { + // Invalid token errors... + _removeToken({ + apn: notification.token, + }); + } + })); + + initFeedback({ options, _removeToken }); +}; diff --git a/app/push/server/gcm.js b/app/push/server/gcm.js new file mode 100644 index 0000000000000..3bbcf5845138c --- /dev/null +++ b/app/push/server/gcm.js @@ -0,0 +1,140 @@ +/* eslint-disable new-cap */ +import { Match } from 'meteor/check'; +import { EJSON } from 'meteor/ejson'; +import _ from 'underscore'; +import gcm from 'node-gcm'; +import Fiber from 'fibers'; + +import { logger } from './logger'; + +export const sendGCM = function({ userTokens, notification, _replaceToken, _removeToken, options }) { + if (Match.test(notification.gcm, Object)) { + notification = _.extend({}, notification, notification.gcm); + } + + // Make sure userTokens are an array of strings + if (userTokens === `${ userTokens }`) { + userTokens = [userTokens]; + } + + // Check if any tokens in there to send + if (!userTokens.length) { + logger.debug('sendGCM no push tokens found'); + return; + } + + logger.debug('sendGCM', userTokens, notification); + + // Allow user to set payload + const data = notification.payload ? { ejson: EJSON.stringify(notification.payload) } : {}; + + data.title = notification.title; + data.message = notification.text; + + // Set image + if (typeof notification.image !== 'undefined') { + data.image = notification.image; + } + + // Set extra details + if (typeof notification.badge !== 'undefined') { + data.msgcnt = notification.badge; + } + if (typeof notification.sound !== 'undefined') { + data.soundname = notification.sound; + } + if (typeof notification.notId !== 'undefined') { + data.notId = notification.notId; + } + if (typeof notification.style !== 'undefined') { + data.style = notification.style; + } + if (typeof notification.summaryText !== 'undefined') { + data.summaryText = notification.summaryText; + } + if (typeof notification.picture !== 'undefined') { + data.picture = notification.picture; + } + + // var message = new gcm.Message(); + const message = new gcm.Message({ + collapseKey: notification.from, + // delayWhileIdle: true, + // timeToLive: 4, + // restricted_package_name: 'dk.gi2.app' + data, + }); + + logger.debug(`Create GCM Sender using "${ options.gcm.apiKey }"`); + const sender = new gcm.Sender(options.gcm.apiKey); + + _.each(userTokens, function(value /* , key */) { + logger.debug(`A:Send message to: ${ value }`); + }); + + /* message.addData('title', title); + message.addData('message', text); + message.addData('msgcnt', '1'); + message.collapseKey = 'sitDrift'; + message.delayWhileIdle = true; + message.timeToLive = 3;*/ + + // /** + // * Parameters: message-literal, userTokens-array, No. of retries, callback-function + // */ + + const userToken = userTokens.length === 1 ? userTokens[0] : null; + + sender.send(message, userTokens, 5, function(err, result) { + if (err) { + logger.debug(`ANDROID ERROR: result of sender: ${ result }`); + return; + } + + if (result === null) { + logger.debug('ANDROID: Result of sender is null'); + return; + } + + logger.debuglog(`ANDROID: Result of sender: ${ JSON.stringify(result) }`); + + if (result.canonical_ids === 1 && userToken) { // jshint ignore:line + // This is an old device, token is replaced + Fiber(function(self) { + // Run in fiber + try { + self.callback(self.oldToken, self.newToken); + } catch (err) { + // + } + }).run({ + oldToken: { gcm: userToken }, + newToken: { gcm: result.results[0].registration_id }, // jshint ignore:line + callback: _replaceToken, + }); + // _replaceToken({ gcm: userToken }, { gcm: result.results[0].registration_id }); + } + // We cant send to that token - might not be registred + // ask the user to remove the token from the list + if (result.failure !== 0 && userToken) { + // This is an old device, token is replaced + Fiber(function(self) { + // Run in fiber + try { + self.callback(self.token); + } catch (err) { + // + } + }).run({ + token: { gcm: userToken }, + callback: _removeToken, + }); + // _replaceToken({ gcm: userToken }, { gcm: result.results[0].registration_id }); + } + }); + // /** Use the following line if you want to send the message without retries + // sender.sendNoRetry(message, userTokens, function (result) { + // console.log('ANDROID: ' + JSON.stringify(result)); + // }); + // **/ +}; // EO sendAndroid diff --git a/app/push/server/logger.js b/app/push/server/logger.js new file mode 100644 index 0000000000000..553e253eee470 --- /dev/null +++ b/app/push/server/logger.js @@ -0,0 +1,4 @@ +import { Logger, LoggerManager } from '../../logger/server'; + +export const logger = new Logger('Push'); +export { LoggerManager }; diff --git a/app/push/server/notifications.js b/app/push/server/notifications.js new file mode 100644 index 0000000000000..bef5f8085db75 --- /dev/null +++ b/app/push/server/notifications.js @@ -0,0 +1,109 @@ +import { Match, check } from 'meteor/check'; +import _ from 'underscore'; + +import { _matchToken, notificationsCollection } from './push'; + +// This is a general function to validate that the data added to notifications +// is in the correct format. If not this function will throw errors +const _validateDocument = function(notification) { + // Check the general notification + check(notification, { + from: String, + title: String, + text: String, + sent: Match.Optional(Boolean), + sending: Match.Optional(Match.Integer), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + contentAvailable: Match.Optional(Match.Integer), + apn: Match.Optional({ + from: Match.Optional(String), + title: Match.Optional(String), + text: Match.Optional(String), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + category: Match.Optional(String), + }), + gcm: Match.Optional({ + from: Match.Optional(String), + title: Match.Optional(String), + text: Match.Optional(String), + image: Match.Optional(String), + style: Match.Optional(String), + summaryText: Match.Optional(String), + picture: Match.Optional(String), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + }), + query: Match.Optional(String), + token: Match.Optional(_matchToken), + tokens: Match.Optional([_matchToken]), + payload: Match.Optional(Object), + delayUntil: Match.Optional(Date), + createdAt: Date, + createdBy: Match.OneOf(String, null), + }); + + // Make sure a token selector or query have been set + if (!notification.token && !notification.tokens && !notification.query) { + throw new Error('No token selector or query found'); + } + + // If tokens array is set it should not be empty + if (notification.tokens && !notification.tokens.length) { + throw new Error('No tokens in array'); + } +}; + +export const send = function(options) { + // If on the client we set the user id - on the server we need an option + // set or we default to "" as the creator of the notification + // If current user not set see if we can set it to the logged in user + // this will only run on the client if Meteor.userId is available + const currentUser = options.createdBy || ''; + + // Rig the notification object + const notification = _.extend({ + createdAt: new Date(), + createdBy: currentUser, + }, _.pick(options, 'from', 'title', 'text')); + + // Add extra + _.extend(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil')); + + if (Match.test(options.apn, Object)) { + notification.apn = _.pick(options.apn, 'from', 'title', 'text', 'badge', 'sound', 'notId', 'category'); + } + + if (Match.test(options.gcm, Object)) { + notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId'); + } + + // Set one token selector, this can be token, array of tokens or query + if (options.query) { + // Set query to the json string version fixing #43 and #39 + notification.query = JSON.stringify(options.query); + } else if (options.token) { + // Set token + notification.token = options.token; + } else if (options.tokens) { + // Set tokens + notification.tokens = options.tokens; + } + // console.log(options); + if (typeof options.contentAvailable !== 'undefined') { + notification.contentAvailable = options.contentAvailable; + } + + notification.sent = false; + notification.sending = 0; + + // Validate the notification + _validateDocument(notification); + + // Try to add the notification to send, we return an id to keep track + return notificationsCollection.insert(notification); +}; diff --git a/app/push/server/push.api.js b/app/push/server/push.api.js new file mode 100644 index 0000000000000..438a1b671dfb3 --- /dev/null +++ b/app/push/server/push.api.js @@ -0,0 +1,318 @@ +/* eslint-disable new-cap */ +import { Meteor } from 'meteor/meteor'; +import _ from 'underscore'; + +import { appTokensCollection, notificationsCollection } from './push'; +import { initAPN, sendAPN } from './apn'; +import { sendGCM } from './gcm'; +import { logger, LoggerManager } from './logger'; + +let isConfigured = false; + +const sendWorker = function(task, interval) { + logger.debug(`Send worker started, using interval: ${ interval }`); + + return Meteor.setInterval(function() { + // xxx: add exponential backoff on error + try { + task(); + } catch (error) { + logger.debug(`Error while sending: ${ error.message }`); + } + }, interval); +}; + +export const Configure = function(options) { + options = _.extend({ + sendTimeout: 60000, // Timeout period for notification send + }, options); + // https://npmjs.org/package/apn + + // After requesting the certificate from Apple, export your private key as + // a .p12 file anddownload the .cer file from the iOS Provisioning Portal. + + // gateway.push.apple.com, port 2195 + // gateway.sandbox.push.apple.com, port 2195 + + // Now, in the directory containing cert.cer and key.p12 execute the + // following commands to generate your .pem files: + // $ openssl x509 -in cert.cer -inform DER -outform PEM -out cert.pem + // $ openssl pkcs12 -in key.p12 -out key.pem -nodes + + // Block multiple calls + if (isConfigured) { + throw new Error('Configure should not be called more than once!'); + } + + isConfigured = true; + + logger.debug('Configure', options); + + const _replaceToken = function(currentToken, newToken) { + appTokensCollection.update({ token: currentToken }, { $set: { token: newToken } }, { multi: true }); + }; + + const _removeToken = function(token) { + appTokensCollection.update({ token }, { $unset: { token: true } }, { multi: true }); + }; + + if (options.apn) { + initAPN({ options, _removeToken }); + } // EO ios notification + + // Universal send function + const _querySend = function(query, notification) { + const countApn = []; + const countGcm = []; + + appTokensCollection.find(query).forEach(function(app) { + logger.debug('send to token', app.token); + + if (app.token.apn) { + countApn.push(app._id); + // Send to APN + if (options.apn) { + sendAPN(app.token.apn, notification); + } + } else if (app.token.gcm) { + countGcm.push(app._id); + + // Send to GCM + // We do support multiple here - so we should construct an array + // and send it bulk - Investigate limit count of id's + if (options.gcm && options.gcm.apiKey) { + sendGCM({ userTokens: app.token.gcm, notification, _replaceToken, _removeToken, options }); + } + } else { + throw new Error('send got a faulty query'); + } + }); + + if (LoggerManager.logLevel === 2) { + logger.debug(`Sent message "${ notification.title }" to ${ countApn.length } ios apps ${ countGcm.length } android apps`); + + // Add some verbosity about the send result, making sure the developer + // understands what just happened. + if (!countApn.length && !countGcm.length) { + if (appTokensCollection.find().count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" is empty - No clients have registered on the server yet...'); + } + } else if (!countApn.length) { + if (appTokensCollection.find({ 'token.apn': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No APN clients have registered on the server yet...'); + } + } else if (!countGcm.length) { + if (appTokensCollection.find({ 'token.gcm': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No GCM clients have registered on the server yet...'); + } + } + } + + return { + apn: countApn, + gcm: countGcm, + }; + }; + + const serverSend = function(options) { + options = options || { badge: 0 }; + let query; + + // Check basic options + if (options.from !== `${ options.from }`) { + throw new Error('send: option "from" not a string'); + } + + if (options.title !== `${ options.title }`) { + throw new Error('send: option "title" not a string'); + } + + if (options.text !== `${ options.text }`) { + throw new Error('send: option "text" not a string'); + } + + if (options.token || options.tokens) { + // The user set one token or array of tokens + const tokenList = options.token ? [options.token] : options.tokens; + + logger.debug(`Send message "${ options.title }" via token(s)`, tokenList); + + query = { + $or: [ + // XXX: Test this query: can we hand in a list of push tokens? + { $and: [ + { token: { $in: tokenList } }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }, + // XXX: Test this query: does this work on app id? + { $and: [ + { _id: { $in: tokenList } }, // one of the app ids + { $or: [ + { 'token.apn': { $exists: true } }, // got apn token + { 'token.gcm': { $exists: true } }, // got gcm token + ] }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }, + ], + }; + } else if (options.query) { + logger.debug(`Send message "${ options.title }" via query`, options.query); + + query = { + $and: [ + options.query, // query object + { $or: [ + { 'token.apn': { $exists: true } }, // got apn token + { 'token.gcm': { $exists: true } }, // got gcm token + ] }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }; + } + + + if (query) { + // Convert to querySend and return status + return _querySend(query, options); + } + throw new Error('send: please set option "token"/"tokens" or "query"'); + }; + + + // This interval will allow only one notification to be sent at a time, it + // will check for new notifications at every `options.sendInterval` + // (default interval is 15000 ms) + // + // It looks in notifications collection to see if theres any pending + // notifications, if so it will try to reserve the pending notification. + // If successfully reserved the send is started. + // + // If notification.query is type string, it's assumed to be a json string + // version of the query selector. Making it able to carry `$` properties in + // the mongo collection. + // + // Pr. default notifications are removed from the collection after send have + // completed. Setting `options.keepNotifications` will update and keep the + // notification eg. if needed for historical reasons. + // + // After the send have completed a "send" event will be emitted with a + // status object containing notification id and the send result object. + // + let isSendingNotification = false; + + if (options.sendInterval !== null) { + // This will require index since we sort notifications by createdAt + notificationsCollection._ensureIndex({ createdAt: 1 }); + notificationsCollection._ensureIndex({ sent: 1 }); + notificationsCollection._ensureIndex({ sending: 1 }); + notificationsCollection._ensureIndex({ delayUntil: 1 }); + + const sendNotification = function(notification) { + // Reserve notification + const now = +new Date(); + const timeoutAt = now + options.sendTimeout; + const reserved = notificationsCollection.update({ + _id: notification._id, + sent: false, // xxx: need to make sure this is set on create + sending: { $lt: now }, + }, + { + $set: { + sending: timeoutAt, + }, + }); + + // Make sure we only handle notifications reserved by this + // instance + if (reserved) { + // Check if query is set and is type String + if (notification.query && notification.query === `${ notification.query }`) { + try { + // The query is in string json format - we need to parse it + notification.query = JSON.parse(notification.query); + } catch (err) { + // Did the user tamper with this?? + throw new Error(`Error while parsing query string, Error: ${ err.message }`); + } + } + + // Send the notification + const result = serverSend(notification); + + if (!options.keepNotifications) { + // Pr. Default we will remove notifications + notificationsCollection.remove({ _id: notification._id }); + } else { + // Update the notification + notificationsCollection.update({ _id: notification._id }, { + $set: { + // Mark as sent + sent: true, + // Set the sent date + sentAt: new Date(), + // Count + count: result, + // Not being sent anymore + sending: 0, + }, + }); + } + + // Emit the send + // self.emit('send', { notification: notification._id, result }); + } // Else could not reserve + }; // EO sendNotification + + sendWorker(function() { + if (isSendingNotification) { + return; + } + + try { + // Set send fence + isSendingNotification = true; + + // var countSent = 0; + const batchSize = options.sendBatchSize || 1; + + const now = +new Date(); + + // Find notifications that are not being or already sent + const pendingNotifications = notificationsCollection.find({ $and: [ + // Message is not sent + { sent: false }, + // And not being sent by other instances + { sending: { $lt: now } }, + // And not queued for future + { $or: [ + { delayUntil: { $exists: false } }, + { delayUntil: { $lte: new Date() } }, + ], + }, + ] }, { + // Sort by created date + sort: { createdAt: 1 }, + limit: batchSize, + }); + + pendingNotifications.forEach(function(notification) { + try { + sendNotification(notification); + } catch (error) { + logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); + } + }); // EO forEach + } finally { + // Remove the send fence + isSendingNotification = false; + } + }, options.sendInterval || 15000); // Default every 15th sec + } else { + logger.debug('Send server is disabled'); + } +}; diff --git a/app/push/server/push.js b/app/push/server/push.js new file mode 100644 index 0000000000000..93198c96053d7 --- /dev/null +++ b/app/push/server/push.js @@ -0,0 +1,7 @@ +import { Match } from 'meteor/check'; +import { Mongo } from 'meteor/mongo'; + +export const _matchToken = Match.OneOf({ apn: String }, { gcm: String }); +export const notificationsCollection = new Mongo.Collection('_raix_push_notifications'); +export const appTokensCollection = new Mongo.Collection('_raix_push_app_tokens'); +appTokensCollection._ensureIndex({ userId: 1 }); diff --git a/app/push/server/server.js b/app/push/server/server.js new file mode 100644 index 0000000000000..beb7604d03321 --- /dev/null +++ b/app/push/server/server.js @@ -0,0 +1,149 @@ +import { Meteor } from 'meteor/meteor'; +import { Match, check } from 'meteor/check'; +import { Random } from 'meteor/random'; + +import { _matchToken, appTokensCollection } from './push'; +import { logger } from './logger'; + +Meteor.methods({ + 'raix:push-update'(options) { + logger.debug('Got push token from app:', options); + + check(options, { + id: Match.Optional(String), + token: _matchToken, + appName: String, + userId: Match.OneOf(String, null), + metadata: Match.Optional(Object), + }); + + // The if user id is set then user id should match on client and connection + if (options.userId && options.userId !== this.userId) { + throw new Meteor.Error(403, 'Forbidden access'); + } + + let doc; + + // lookup app by id if one was included + if (options.id) { + doc = appTokensCollection.findOne({ _id: options.id }); + } else if (options.userId) { + doc = appTokensCollection.findOne({ userId: options.userId }); + } + + // No doc was found - we check the database to see if + // we can find a match for the app via token and appName + if (!doc) { + doc = appTokensCollection.findOne({ + $and: [ + { token: options.token }, // Match token + { appName: options.appName }, // Match appName + { token: { $exists: true } }, // Make sure token exists + ], + }); + } + + // if we could not find the id or token then create it + if (!doc) { + // Rig default doc + doc = { + token: options.token, + appName: options.appName, + userId: options.userId, + enabled: true, + createdAt: new Date(), + updatedAt: new Date(), + }; + + // XXX: We might want to check the id - Why isnt there a match for id + // in the Meteor check... Normal length 17 (could be larger), and + // numbers+letters are used in Random.id() with exception of 0 and 1 + doc._id = options.id || Random.id(); + // The user wanted us to use a specific id, we didn't find this while + // searching. The client could depend on the id eg. as reference so + // we respect this and try to create a document with the selected id; + appTokensCollection._collection.insert(doc); + } else { + // We found the app so update the updatedAt and set the token + appTokensCollection.update({ _id: doc._id }, { + $set: { + updatedAt: new Date(), + token: options.token, + }, + }); + } + + if (doc) { + // xxx: Hack + // Clean up mech making sure tokens are uniq - android sometimes generate + // new tokens resulting in duplicates + const removed = appTokensCollection.remove({ + $and: [ + { _id: { $ne: doc._id } }, + { token: doc.token }, // Match token + { appName: doc.appName }, // Match appName + { token: { $exists: true } }, // Make sure token exists + ], + }); + + if (removed) { + logger.debug(`Removed ${ removed } existing app items`); + } + } + + if (doc) { + logger.debug('updated', doc); + } + + if (!doc) { + throw new Meteor.Error(500, 'setPushToken could not create record'); + } + // Return the doc we want to use + return doc; + }, + 'raix:push-setuser'(id) { + check(id, String); + + logger.debug(`Settings userId "${ this.userId }" for app:`, id); + // We update the appCollection id setting the Meteor.userId + const found = appTokensCollection.update({ _id: id }, { $set: { userId: this.userId } }); + + // Note that the app id might not exist because no token is set yet. + // We do create the new app id for the user since we might store additional + // metadata for the app / user + + // If id not found then create it? + // We dont, its better to wait until the user wants to + // store metadata or token - We could end up with unused data in the + // collection at every app re-install / update + // + // The user could store some metadata in appCollectin but only if they + // have created the app and provided a token. + // If not the metadata should be set via ground:db + + return !!found; + }, + 'raix:push-metadata'(data) { + check(data, { + id: String, + metadata: Object, + }); + + // Set the metadata + const found = appTokensCollection.update({ _id: data.id }, { $set: { metadata: data.metadata } }); + + return !!found; + }, + 'raix:push-enable'(data) { + check(data, { + id: String, + enabled: Boolean, + }); + + logger.debug(`Setting enabled to "${ data.enabled }" for app:`, data.id); + + const found = appTokensCollection.update({ _id: data.id }, { $set: { enabled: data.enabled } }); + + return !!found; + }, +}); diff --git a/app/statistics/server/lib/statistics.js b/app/statistics/server/lib/statistics.js index b2907cd0ad817..331bb110dc789 100644 --- a/app/statistics/server/lib/statistics.js +++ b/app/statistics/server/lib/statistics.js @@ -1,10 +1,10 @@ import os from 'os'; import _ from 'underscore'; -import { Push } from 'meteor/rocketchat:push'; import { Meteor } from 'meteor/meteor'; import { InstanceStatus } from 'meteor/konecty:multiple-instances-status'; +import { notificationsCollection } from '../../../push/server/push'; import { Sessions, Settings, @@ -166,7 +166,7 @@ export const statistics = { totalWithScriptEnabled: integrations.filter((integration) => integration.scriptEnabled === true).length, }; - statistics.pushQueue = Push.notifications.find().count(); + statistics.pushQueue = notificationsCollection.find().count(); return statistics; }, diff --git a/package-lock.json b/package-lock.json index 2ace3c2d27024..85a093b274b90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7034,6 +7034,21 @@ "integrity": "sha512-/FQM1EDkTsf63Ub2C6O7GuYFDsSXUwsaZDurV0np41ocwq0jthUAYCmhBX9f+KwlaCgIuWyr/4WlUQUBfKfZog==", "dev": true }, + "apn": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/apn/-/apn-1.6.2.tgz", + "integrity": "sha1-wHTUEiC9t+ahlQIHXU/roZaDg/M=", + "requires": { + "q": "1.x" + }, + "dependencies": { + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" + } + } + }, "app-root-dir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/app-root-dir/-/app-root-dir-1.0.2.tgz", @@ -22379,6 +22394,28 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", "integrity": "sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw==" }, + "node-gcm": { + "version": "0.14.4", + "resolved": "https://registry.npmjs.org/node-gcm/-/node-gcm-0.14.4.tgz", + "integrity": "sha1-mWXbzjcEcuFbGGPovEtTovr/qQM=", + "requires": { + "debug": "^0.8.1", + "lodash": "^3.10.1", + "request": "^2.27.0" + }, + "dependencies": { + "debug": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-0.8.1.tgz", + "integrity": "sha1-IP9NJvXkIstoobrLu2EDmtjBwTA=" + }, + "lodash": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", + "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=" + } + } + }, "node-libs-browser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.0.tgz", diff --git a/package.json b/package.json index 83e34c5f96ed1..bf4cc33e3ee02 100644 --- a/package.json +++ b/package.json @@ -137,6 +137,7 @@ "@rocket.chat/ui-kit": "^0.7.1", "@slack/client": "^4.8.0", "adm-zip": "RocketChat/adm-zip", + "apn": "1.6.2", "archiver": "^3.0.0", "arraybuffer-to-string": "^1.0.2", "atlassian-crowd": "^0.5.0", @@ -201,6 +202,7 @@ "moment": "^2.22.2", "moment-timezone": "^0.5.27", "node-dogstatsd": "^0.0.7", + "node-gcm": "0.14.4", "node-rsa": "^1.0.5", "object-path": "^0.11.4", "pdfjs-dist": "^2.0.943", diff --git a/server/lib/pushConfig.js b/server/lib/pushConfig.js index 5bdce4ec40ccd..52fab8d894d13 100644 --- a/server/lib/pushConfig.js +++ b/server/lib/pushConfig.js @@ -7,6 +7,7 @@ import { SystemLogger } from '../../app/logger'; import { getWorkspaceAccessToken } from '../../app/cloud/server'; import { hasRole } from '../../app/authorization'; import { settings } from '../../app/settings'; +import { appTokensCollection } from '../../app/push/server/push'; Meteor.methods({ @@ -51,7 +52,7 @@ Meteor.methods({ }], }; - const tokens = Push.appCollection.find(query).count(); + const tokens = appTokensCollection.find(query).count(); if (tokens === 0) { throw new Meteor.Error('error-no-tokens-for-this-user', 'There are no tokens for this user', { @@ -98,7 +99,7 @@ function sendPush(gateway, service, token, options, tries = 0) { return HTTP.post(`${ gateway }/push/${ service }/send`, data, function(error, response) { if (response && response.statusCode === 406) { console.log('removing push token', token); - Push.appCollection.remove({ + appTokensCollection.remove({ $or: [{ 'token.apn': token, }, { @@ -212,7 +213,7 @@ function configurePush() { }], }; - Push.appCollection.find(query).forEach((app) => { + appTokensCollection.find(query).forEach((app) => { if (settings.get('Push_debug')) { console.log('Push: send to token', app.token); } diff --git a/server/startup/migrations/v181.js b/server/startup/migrations/v181.js index bdcf8fbfd08cd..d0379ecf72caf 100644 --- a/server/startup/migrations/v181.js +++ b/server/startup/migrations/v181.js @@ -1,5 +1,4 @@ -import { Push } from 'meteor/rocketchat:push'; - +import { notificationsCollection } from '../../../app/push/server/push'; import { Migrations } from '../../../app/migrations/server'; import { Settings } from '../../../app/models/server'; @@ -13,6 +12,6 @@ Migrations.add({ date.setHours(date.getHours() - 2); // 2 hours ago; // Remove all records older than 2h - Push.notifications.rawCollection().removeMany({ createdAt: { $lt: date } }); + notificationsCollection.rawCollection().removeMany({ createdAt: { $lt: date } }); }, }); From df9003c26c76fbf1a4733da6fdf44d05ca491301 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 16 Apr 2020 22:01:25 -0300 Subject: [PATCH 02/40] Simplify imported lib --- app/push/server/{server.js => methods.js} | 0 app/push/server/notifications.js | 109 ------ app/push/server/push.api.js | 318 ---------------- app/push/server/push.js | 423 +++++++++++++++++++++- 4 files changed, 422 insertions(+), 428 deletions(-) rename app/push/server/{server.js => methods.js} (100%) delete mode 100644 app/push/server/notifications.js delete mode 100644 app/push/server/push.api.js diff --git a/app/push/server/server.js b/app/push/server/methods.js similarity index 100% rename from app/push/server/server.js rename to app/push/server/methods.js diff --git a/app/push/server/notifications.js b/app/push/server/notifications.js deleted file mode 100644 index bef5f8085db75..0000000000000 --- a/app/push/server/notifications.js +++ /dev/null @@ -1,109 +0,0 @@ -import { Match, check } from 'meteor/check'; -import _ from 'underscore'; - -import { _matchToken, notificationsCollection } from './push'; - -// This is a general function to validate that the data added to notifications -// is in the correct format. If not this function will throw errors -const _validateDocument = function(notification) { - // Check the general notification - check(notification, { - from: String, - title: String, - text: String, - sent: Match.Optional(Boolean), - sending: Match.Optional(Match.Integer), - badge: Match.Optional(Match.Integer), - sound: Match.Optional(String), - notId: Match.Optional(Match.Integer), - contentAvailable: Match.Optional(Match.Integer), - apn: Match.Optional({ - from: Match.Optional(String), - title: Match.Optional(String), - text: Match.Optional(String), - badge: Match.Optional(Match.Integer), - sound: Match.Optional(String), - notId: Match.Optional(Match.Integer), - category: Match.Optional(String), - }), - gcm: Match.Optional({ - from: Match.Optional(String), - title: Match.Optional(String), - text: Match.Optional(String), - image: Match.Optional(String), - style: Match.Optional(String), - summaryText: Match.Optional(String), - picture: Match.Optional(String), - badge: Match.Optional(Match.Integer), - sound: Match.Optional(String), - notId: Match.Optional(Match.Integer), - }), - query: Match.Optional(String), - token: Match.Optional(_matchToken), - tokens: Match.Optional([_matchToken]), - payload: Match.Optional(Object), - delayUntil: Match.Optional(Date), - createdAt: Date, - createdBy: Match.OneOf(String, null), - }); - - // Make sure a token selector or query have been set - if (!notification.token && !notification.tokens && !notification.query) { - throw new Error('No token selector or query found'); - } - - // If tokens array is set it should not be empty - if (notification.tokens && !notification.tokens.length) { - throw new Error('No tokens in array'); - } -}; - -export const send = function(options) { - // If on the client we set the user id - on the server we need an option - // set or we default to "" as the creator of the notification - // If current user not set see if we can set it to the logged in user - // this will only run on the client if Meteor.userId is available - const currentUser = options.createdBy || ''; - - // Rig the notification object - const notification = _.extend({ - createdAt: new Date(), - createdBy: currentUser, - }, _.pick(options, 'from', 'title', 'text')); - - // Add extra - _.extend(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil')); - - if (Match.test(options.apn, Object)) { - notification.apn = _.pick(options.apn, 'from', 'title', 'text', 'badge', 'sound', 'notId', 'category'); - } - - if (Match.test(options.gcm, Object)) { - notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId'); - } - - // Set one token selector, this can be token, array of tokens or query - if (options.query) { - // Set query to the json string version fixing #43 and #39 - notification.query = JSON.stringify(options.query); - } else if (options.token) { - // Set token - notification.token = options.token; - } else if (options.tokens) { - // Set tokens - notification.tokens = options.tokens; - } - // console.log(options); - if (typeof options.contentAvailable !== 'undefined') { - notification.contentAvailable = options.contentAvailable; - } - - notification.sent = false; - notification.sending = 0; - - // Validate the notification - _validateDocument(notification); - - // Try to add the notification to send, we return an id to keep track - return notificationsCollection.insert(notification); -}; diff --git a/app/push/server/push.api.js b/app/push/server/push.api.js deleted file mode 100644 index 438a1b671dfb3..0000000000000 --- a/app/push/server/push.api.js +++ /dev/null @@ -1,318 +0,0 @@ -/* eslint-disable new-cap */ -import { Meteor } from 'meteor/meteor'; -import _ from 'underscore'; - -import { appTokensCollection, notificationsCollection } from './push'; -import { initAPN, sendAPN } from './apn'; -import { sendGCM } from './gcm'; -import { logger, LoggerManager } from './logger'; - -let isConfigured = false; - -const sendWorker = function(task, interval) { - logger.debug(`Send worker started, using interval: ${ interval }`); - - return Meteor.setInterval(function() { - // xxx: add exponential backoff on error - try { - task(); - } catch (error) { - logger.debug(`Error while sending: ${ error.message }`); - } - }, interval); -}; - -export const Configure = function(options) { - options = _.extend({ - sendTimeout: 60000, // Timeout period for notification send - }, options); - // https://npmjs.org/package/apn - - // After requesting the certificate from Apple, export your private key as - // a .p12 file anddownload the .cer file from the iOS Provisioning Portal. - - // gateway.push.apple.com, port 2195 - // gateway.sandbox.push.apple.com, port 2195 - - // Now, in the directory containing cert.cer and key.p12 execute the - // following commands to generate your .pem files: - // $ openssl x509 -in cert.cer -inform DER -outform PEM -out cert.pem - // $ openssl pkcs12 -in key.p12 -out key.pem -nodes - - // Block multiple calls - if (isConfigured) { - throw new Error('Configure should not be called more than once!'); - } - - isConfigured = true; - - logger.debug('Configure', options); - - const _replaceToken = function(currentToken, newToken) { - appTokensCollection.update({ token: currentToken }, { $set: { token: newToken } }, { multi: true }); - }; - - const _removeToken = function(token) { - appTokensCollection.update({ token }, { $unset: { token: true } }, { multi: true }); - }; - - if (options.apn) { - initAPN({ options, _removeToken }); - } // EO ios notification - - // Universal send function - const _querySend = function(query, notification) { - const countApn = []; - const countGcm = []; - - appTokensCollection.find(query).forEach(function(app) { - logger.debug('send to token', app.token); - - if (app.token.apn) { - countApn.push(app._id); - // Send to APN - if (options.apn) { - sendAPN(app.token.apn, notification); - } - } else if (app.token.gcm) { - countGcm.push(app._id); - - // Send to GCM - // We do support multiple here - so we should construct an array - // and send it bulk - Investigate limit count of id's - if (options.gcm && options.gcm.apiKey) { - sendGCM({ userTokens: app.token.gcm, notification, _replaceToken, _removeToken, options }); - } - } else { - throw new Error('send got a faulty query'); - } - }); - - if (LoggerManager.logLevel === 2) { - logger.debug(`Sent message "${ notification.title }" to ${ countApn.length } ios apps ${ countGcm.length } android apps`); - - // Add some verbosity about the send result, making sure the developer - // understands what just happened. - if (!countApn.length && !countGcm.length) { - if (appTokensCollection.find().count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" is empty - No clients have registered on the server yet...'); - } - } else if (!countApn.length) { - if (appTokensCollection.find({ 'token.apn': { $exists: true } }).count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" - No APN clients have registered on the server yet...'); - } - } else if (!countGcm.length) { - if (appTokensCollection.find({ 'token.gcm': { $exists: true } }).count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" - No GCM clients have registered on the server yet...'); - } - } - } - - return { - apn: countApn, - gcm: countGcm, - }; - }; - - const serverSend = function(options) { - options = options || { badge: 0 }; - let query; - - // Check basic options - if (options.from !== `${ options.from }`) { - throw new Error('send: option "from" not a string'); - } - - if (options.title !== `${ options.title }`) { - throw new Error('send: option "title" not a string'); - } - - if (options.text !== `${ options.text }`) { - throw new Error('send: option "text" not a string'); - } - - if (options.token || options.tokens) { - // The user set one token or array of tokens - const tokenList = options.token ? [options.token] : options.tokens; - - logger.debug(`Send message "${ options.title }" via token(s)`, tokenList); - - query = { - $or: [ - // XXX: Test this query: can we hand in a list of push tokens? - { $and: [ - { token: { $in: tokenList } }, - // And is not disabled - { enabled: { $ne: false } }, - ], - }, - // XXX: Test this query: does this work on app id? - { $and: [ - { _id: { $in: tokenList } }, // one of the app ids - { $or: [ - { 'token.apn': { $exists: true } }, // got apn token - { 'token.gcm': { $exists: true } }, // got gcm token - ] }, - // And is not disabled - { enabled: { $ne: false } }, - ], - }, - ], - }; - } else if (options.query) { - logger.debug(`Send message "${ options.title }" via query`, options.query); - - query = { - $and: [ - options.query, // query object - { $or: [ - { 'token.apn': { $exists: true } }, // got apn token - { 'token.gcm': { $exists: true } }, // got gcm token - ] }, - // And is not disabled - { enabled: { $ne: false } }, - ], - }; - } - - - if (query) { - // Convert to querySend and return status - return _querySend(query, options); - } - throw new Error('send: please set option "token"/"tokens" or "query"'); - }; - - - // This interval will allow only one notification to be sent at a time, it - // will check for new notifications at every `options.sendInterval` - // (default interval is 15000 ms) - // - // It looks in notifications collection to see if theres any pending - // notifications, if so it will try to reserve the pending notification. - // If successfully reserved the send is started. - // - // If notification.query is type string, it's assumed to be a json string - // version of the query selector. Making it able to carry `$` properties in - // the mongo collection. - // - // Pr. default notifications are removed from the collection after send have - // completed. Setting `options.keepNotifications` will update and keep the - // notification eg. if needed for historical reasons. - // - // After the send have completed a "send" event will be emitted with a - // status object containing notification id and the send result object. - // - let isSendingNotification = false; - - if (options.sendInterval !== null) { - // This will require index since we sort notifications by createdAt - notificationsCollection._ensureIndex({ createdAt: 1 }); - notificationsCollection._ensureIndex({ sent: 1 }); - notificationsCollection._ensureIndex({ sending: 1 }); - notificationsCollection._ensureIndex({ delayUntil: 1 }); - - const sendNotification = function(notification) { - // Reserve notification - const now = +new Date(); - const timeoutAt = now + options.sendTimeout; - const reserved = notificationsCollection.update({ - _id: notification._id, - sent: false, // xxx: need to make sure this is set on create - sending: { $lt: now }, - }, - { - $set: { - sending: timeoutAt, - }, - }); - - // Make sure we only handle notifications reserved by this - // instance - if (reserved) { - // Check if query is set and is type String - if (notification.query && notification.query === `${ notification.query }`) { - try { - // The query is in string json format - we need to parse it - notification.query = JSON.parse(notification.query); - } catch (err) { - // Did the user tamper with this?? - throw new Error(`Error while parsing query string, Error: ${ err.message }`); - } - } - - // Send the notification - const result = serverSend(notification); - - if (!options.keepNotifications) { - // Pr. Default we will remove notifications - notificationsCollection.remove({ _id: notification._id }); - } else { - // Update the notification - notificationsCollection.update({ _id: notification._id }, { - $set: { - // Mark as sent - sent: true, - // Set the sent date - sentAt: new Date(), - // Count - count: result, - // Not being sent anymore - sending: 0, - }, - }); - } - - // Emit the send - // self.emit('send', { notification: notification._id, result }); - } // Else could not reserve - }; // EO sendNotification - - sendWorker(function() { - if (isSendingNotification) { - return; - } - - try { - // Set send fence - isSendingNotification = true; - - // var countSent = 0; - const batchSize = options.sendBatchSize || 1; - - const now = +new Date(); - - // Find notifications that are not being or already sent - const pendingNotifications = notificationsCollection.find({ $and: [ - // Message is not sent - { sent: false }, - // And not being sent by other instances - { sending: { $lt: now } }, - // And not queued for future - { $or: [ - { delayUntil: { $exists: false } }, - { delayUntil: { $lte: new Date() } }, - ], - }, - ] }, { - // Sort by created date - sort: { createdAt: 1 }, - limit: batchSize, - }); - - pendingNotifications.forEach(function(notification) { - try { - sendNotification(notification); - } catch (error) { - logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); - } - }); // EO forEach - } finally { - // Remove the send fence - isSendingNotification = false; - } - }, options.sendInterval || 15000); // Default every 15th sec - } else { - logger.debug('Send server is disabled'); - } -}; diff --git a/app/push/server/push.js b/app/push/server/push.js index 93198c96053d7..b77607e96520d 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -1,7 +1,428 @@ -import { Match } from 'meteor/check'; +/* eslint-disable new-cap */ +import { Meteor } from 'meteor/meteor'; +import { Match, check } from 'meteor/check'; import { Mongo } from 'meteor/mongo'; +import _ from 'underscore'; + +import { initAPN, sendAPN } from './apn'; +import { sendGCM } from './gcm'; +import { logger, LoggerManager } from './logger'; export const _matchToken = Match.OneOf({ apn: String }, { gcm: String }); export const notificationsCollection = new Mongo.Collection('_raix_push_notifications'); export const appTokensCollection = new Mongo.Collection('_raix_push_app_tokens'); appTokensCollection._ensureIndex({ userId: 1 }); + +let isConfigured = false; + +const sendWorker = function(task, interval) { + logger.debug(`Send worker started, using interval: ${ interval }`); + + return Meteor.setInterval(function() { + // xxx: add exponential backoff on error + try { + task(); + } catch (error) { + logger.debug(`Error while sending: ${ error.message }`); + } + }, interval); +}; + +const _replaceToken = function(currentToken, newToken) { + appTokensCollection.update({ token: currentToken }, { $set: { token: newToken } }, { multi: true }); +}; + +const _removeToken = function(token) { + appTokensCollection.update({ token }, { $unset: { token: true } }, { multi: true }); +}; + +// Universal send function +const _querySend = function(query, notification, options) { + const countApn = []; + const countGcm = []; + + appTokensCollection.find(query).forEach(function(app) { + logger.debug('send to token', app.token); + + if (app.token.apn) { + countApn.push(app._id); + // Send to APN + if (options.apn) { + sendAPN(app.token.apn, notification); + } + } else if (app.token.gcm) { + countGcm.push(app._id); + + // Send to GCM + // We do support multiple here - so we should construct an array + // and send it bulk - Investigate limit count of id's + if (options.gcm && options.gcm.apiKey) { + sendGCM({ userTokens: app.token.gcm, notification, _replaceToken, _removeToken, options }); + } + } else { + throw new Error('send got a faulty query'); + } + }); + + if (LoggerManager.logLevel === 2) { + logger.debug(`Sent message "${ notification.title }" to ${ countApn.length } ios apps ${ countGcm.length } android apps`); + + // Add some verbosity about the send result, making sure the developer + // understands what just happened. + if (!countApn.length && !countGcm.length) { + if (appTokensCollection.find().count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" is empty - No clients have registered on the server yet...'); + } + } else if (!countApn.length) { + if (appTokensCollection.find({ 'token.apn': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No APN clients have registered on the server yet...'); + } + } else if (!countGcm.length) { + if (appTokensCollection.find({ 'token.gcm': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No GCM clients have registered on the server yet...'); + } + } + } + + return { + apn: countApn, + gcm: countGcm, + }; +}; + +const serverSend = function(notification, options) { + notification = notification || { badge: 0 }; + let query; + + // Check basic options + if (notification.from !== `${ notification.from }`) { + throw new Error('send: option "from" not a string'); + } + + if (notification.title !== `${ notification.title }`) { + throw new Error('send: option "title" not a string'); + } + + if (notification.text !== `${ notification.text }`) { + throw new Error('send: option "text" not a string'); + } + + if (notification.token || notification.tokens) { + // The user set one token or array of tokens + const tokenList = notification.token ? [notification.token] : notification.tokens; + + logger.debug(`Send message "${ notification.title }" via token(s)`, tokenList); + + query = { + $or: [ + // XXX: Test this query: can we hand in a list of push tokens? + { $and: [ + { token: { $in: tokenList } }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }, + // XXX: Test this query: does this work on app id? + { $and: [ + { _id: { $in: tokenList } }, // one of the app ids + { $or: [ + { 'token.apn': { $exists: true } }, // got apn token + { 'token.gcm': { $exists: true } }, // got gcm token + ] }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }, + ], + }; + } else if (notification.query) { + logger.debug(`Send message "${ notification.title }" via query`, notification.query); + + query = { + $and: [ + notification.query, // query object + { $or: [ + { 'token.apn': { $exists: true } }, // got apn token + { 'token.gcm': { $exists: true } }, // got gcm token + ] }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }; + } + + if (query) { + // Convert to querySend and return status + return _querySend(query, notification, options); + } + throw new Error('send: please set option "token"/"tokens" or "query"'); +}; + +export const Configure = function(options) { + options = _.extend({ + sendTimeout: 60000, // Timeout period for notification send + }, options); + // https://npmjs.org/package/apn + + // After requesting the certificate from Apple, export your private key as + // a .p12 file anddownload the .cer file from the iOS Provisioning Portal. + + // gateway.push.apple.com, port 2195 + // gateway.sandbox.push.apple.com, port 2195 + + // Now, in the directory containing cert.cer and key.p12 execute the + // following commands to generate your .pem files: + // $ openssl x509 -in cert.cer -inform DER -outform PEM -out cert.pem + // $ openssl pkcs12 -in key.p12 -out key.pem -nodes + + // Block multiple calls + if (isConfigured) { + throw new Error('Configure should not be called more than once!'); + } + + isConfigured = true; + + logger.debug('Configure', options); + + if (options.apn) { + initAPN({ options, _removeToken }); + } // EO ios notification + + // This interval will allow only one notification to be sent at a time, it + // will check for new notifications at every `options.sendInterval` + // (default interval is 15000 ms) + // + // It looks in notifications collection to see if theres any pending + // notifications, if so it will try to reserve the pending notification. + // If successfully reserved the send is started. + // + // If notification.query is type string, it's assumed to be a json string + // version of the query selector. Making it able to carry `$` properties in + // the mongo collection. + // + // Pr. default notifications are removed from the collection after send have + // completed. Setting `options.keepNotifications` will update and keep the + // notification eg. if needed for historical reasons. + // + // After the send have completed a "send" event will be emitted with a + // status object containing notification id and the send result object. + // + let isSendingNotification = false; + + if (options.sendInterval !== null) { + // This will require index since we sort notifications by createdAt + notificationsCollection._ensureIndex({ createdAt: 1 }); + notificationsCollection._ensureIndex({ sent: 1 }); + notificationsCollection._ensureIndex({ sending: 1 }); + notificationsCollection._ensureIndex({ delayUntil: 1 }); + + const sendNotification = function(notification) { + // Reserve notification + const now = +new Date(); + const timeoutAt = now + options.sendTimeout; + const reserved = notificationsCollection.update({ + _id: notification._id, + sent: false, // xxx: need to make sure this is set on create + sending: { $lt: now }, + }, + { + $set: { + sending: timeoutAt, + }, + }); + + // Make sure we only handle notifications reserved by this + // instance + if (reserved) { + // Check if query is set and is type String + if (notification.query && notification.query === `${ notification.query }`) { + try { + // The query is in string json format - we need to parse it + notification.query = JSON.parse(notification.query); + } catch (err) { + // Did the user tamper with this?? + throw new Error(`Error while parsing query string, Error: ${ err.message }`); + } + } + + // Send the notification + const result = serverSend(notification, options); + + if (!options.keepNotifications) { + // Pr. Default we will remove notifications + notificationsCollection.remove({ _id: notification._id }); + } else { + // Update the notification + notificationsCollection.update({ _id: notification._id }, { + $set: { + // Mark as sent + sent: true, + // Set the sent date + sentAt: new Date(), + // Count + count: result, + // Not being sent anymore + sending: 0, + }, + }); + } + + // Emit the send + // self.emit('send', { notification: notification._id, result }); + } // Else could not reserve + }; // EO sendNotification + + sendWorker(function() { + if (isSendingNotification) { + return; + } + + try { + // Set send fence + isSendingNotification = true; + + // var countSent = 0; + const batchSize = options.sendBatchSize || 1; + + const now = +new Date(); + + // Find notifications that are not being or already sent + const pendingNotifications = notificationsCollection.find({ $and: [ + // Message is not sent + { sent: false }, + // And not being sent by other instances + { sending: { $lt: now } }, + // And not queued for future + { $or: [ + { delayUntil: { $exists: false } }, + { delayUntil: { $lte: new Date() } }, + ], + }, + ] }, { + // Sort by created date + sort: { createdAt: 1 }, + limit: batchSize, + }); + + pendingNotifications.forEach(function(notification) { + try { + sendNotification(notification); + } catch (error) { + logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); + } + }); // EO forEach + } finally { + // Remove the send fence + isSendingNotification = false; + } + }, options.sendInterval || 15000); // Default every 15th sec + } else { + logger.debug('Send server is disabled'); + } +}; + + +// This is a general function to validate that the data added to notifications +// is in the correct format. If not this function will throw errors +const _validateDocument = function(notification) { + // Check the general notification + check(notification, { + from: String, + title: String, + text: String, + sent: Match.Optional(Boolean), + sending: Match.Optional(Match.Integer), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + contentAvailable: Match.Optional(Match.Integer), + apn: Match.Optional({ + from: Match.Optional(String), + title: Match.Optional(String), + text: Match.Optional(String), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + category: Match.Optional(String), + }), + gcm: Match.Optional({ + from: Match.Optional(String), + title: Match.Optional(String), + text: Match.Optional(String), + image: Match.Optional(String), + style: Match.Optional(String), + summaryText: Match.Optional(String), + picture: Match.Optional(String), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + }), + query: Match.Optional(String), + token: Match.Optional(_matchToken), + tokens: Match.Optional([_matchToken]), + payload: Match.Optional(Object), + delayUntil: Match.Optional(Date), + createdAt: Date, + createdBy: Match.OneOf(String, null), + }); + + // Make sure a token selector or query have been set + if (!notification.token && !notification.tokens && !notification.query) { + throw new Error('No token selector or query found'); + } + + // If tokens array is set it should not be empty + if (notification.tokens && !notification.tokens.length) { + throw new Error('No tokens in array'); + } +}; + +export const send = function(options) { + // If on the client we set the user id - on the server we need an option + // set or we default to "" as the creator of the notification + // If current user not set see if we can set it to the logged in user + // this will only run on the client if Meteor.userId is available + const currentUser = options.createdBy || ''; + + // Rig the notification object + const notification = _.extend({ + createdAt: new Date(), + createdBy: currentUser, + }, _.pick(options, 'from', 'title', 'text')); + + // Add extra + _.extend(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil')); + + if (Match.test(options.apn, Object)) { + notification.apn = _.pick(options.apn, 'from', 'title', 'text', 'badge', 'sound', 'notId', 'category'); + } + + if (Match.test(options.gcm, Object)) { + notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId'); + } + + // Set one token selector, this can be token, array of tokens or query + if (options.query) { + // Set query to the json string version fixing #43 and #39 + notification.query = JSON.stringify(options.query); + } else if (options.token) { + // Set token + notification.token = options.token; + } else if (options.tokens) { + // Set tokens + notification.tokens = options.tokens; + } + // console.log(options); + if (typeof options.contentAvailable !== 'undefined') { + notification.contentAvailable = options.contentAvailable; + } + + notification.sent = false; + notification.sending = 0; + + // Validate the notification + _validateDocument(notification); + + // Try to add the notification to send, we return an id to keep track + return notificationsCollection.insert(notification); +}; From 1c205c0f450cd20c55819acd0c0af665e5cb2e09 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 16 Apr 2020 22:24:39 -0300 Subject: [PATCH 03/40] Move our custom push code to inside the library --- app/api/server/v1/push.js | 2 +- .../server/lib/PushNotification.js | 5 +- app/push/server/apn.js | 2 +- app/push/server/index.js | 1 + app/push/server/push.js | 102 +++++++++++++- app/statistics/server/lib/statistics.js | 2 +- server/lib/pushConfig.js | 126 ++---------------- server/startup/migrations/v181.js | 2 +- 8 files changed, 116 insertions(+), 126 deletions(-) create mode 100644 app/push/server/index.js diff --git a/app/api/server/v1/push.js b/app/api/server/v1/push.js index 1d84d57aa5b02..9e3c8d59b325c 100644 --- a/app/api/server/v1/push.js +++ b/app/api/server/v1/push.js @@ -1,7 +1,7 @@ import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; -import { appTokensCollection } from '../../../push/server/push'; +import { appTokensCollection } from '../../../push/server'; import { API } from '../api'; API.v1.addRoute('push.token', { authRequired: true }, { diff --git a/app/push-notifications/server/lib/PushNotification.js b/app/push-notifications/server/lib/PushNotification.js index 25609c54fa91b..fce736ccdaa70 100644 --- a/app/push-notifications/server/lib/PushNotification.js +++ b/app/push-notifications/server/lib/PushNotification.js @@ -1,5 +1,4 @@ -import { Push } from 'meteor/rocketchat:push'; - +import { send } from '../../../push/server'; import { settings } from '../../../settings'; import { metrics } from '../../../metrics'; import { RocketChatAssets } from '../../../assets'; @@ -53,7 +52,7 @@ export class PushNotification { } metrics.notificationsSent.inc({ notification_type: 'mobile' }); - return Push.send(config); + return send(config); } } diff --git a/app/push/server/apn.js b/app/push/server/apn.js index dbb8858260366..fe2440c162646 100644 --- a/app/push/server/apn.js +++ b/app/push/server/apn.js @@ -100,7 +100,7 @@ export const initFeedback = function({ options, _removeToken }) { }; export const initAPN = ({ options, _removeToken }) => { - logger.debug('Push: APN configured'); + logger.debug('APN configured'); // Allow production to be a general option for push notifications if (options.production === Boolean(options.production)) { diff --git a/app/push/server/index.js b/app/push/server/index.js new file mode 100644 index 0000000000000..ae97ba5ec9dab --- /dev/null +++ b/app/push/server/index.js @@ -0,0 +1 @@ +export { send, appTokensCollection, notificationsCollection, configure } from './push'; diff --git a/app/push/server/push.js b/app/push/server/push.js index b77607e96520d..bce84fc832e2a 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -2,6 +2,7 @@ import { Meteor } from 'meteor/meteor'; import { Match, check } from 'meteor/check'; import { Mongo } from 'meteor/mongo'; +import { HTTP } from 'meteor/http'; import _ from 'underscore'; import { initAPN, sendAPN } from './apn'; @@ -90,7 +91,7 @@ const _querySend = function(query, notification, options) { }; }; -const serverSend = function(notification, options) { +const serverSendNative = function(notification, options) { notification = notification || { badge: 0 }; let query; @@ -158,7 +159,104 @@ const serverSend = function(notification, options) { throw new Error('send: please set option "token"/"tokens" or "query"'); }; -export const Configure = function(options) { +const sendGatewayPush = (gateway, service, token, notification, options, tries = 0) => { + notification.uniqueId = options.uniqueId; + + const data = { + data: { + token, + options: notification, + }, + headers: {}, + }; + + if (token && options.getAuthorization) { + data.headers.Authorization = options.getAuthorization(); + } + + return HTTP.post(`${ gateway }/push/${ service }/send`, data, function(error, response) { + if (response && response.statusCode === 406) { + console.log('removing push token', token); + appTokensCollection.remove({ + $or: [{ + 'token.apn': token, + }, { + 'token.gcm': token, + }], + }); + return; + } + + if (!error) { + return; + } + + logger.error(`Error sending push to gateway (${ tries } try) ->`, error); + + if (tries <= 6) { + const ms = Math.pow(10, tries + 2); + + logger.log('Trying sending push to gateway again in', ms, 'milliseconds'); + + return Meteor.setTimeout(function() { + return sendGatewayPush(gateway, service, token, notification, options, tries + 1); + }, ms); + } + }); +}; + +const serverSendGateway = function(notification = { badge: 0 }, options) { + for (const gateway of options.gateways) { + if (notification.from !== String(notification.from)) { + throw new Error('Push.send: option "from" not a string'); + } + if (notification.title !== String(notification.title)) { + throw new Error('Push.send: option "title" not a string'); + } + if (notification.text !== String(notification.text)) { + throw new Error('Push.send: option "text" not a string'); + } + + logger.debug(`send message "${ notification.title }" via query`, notification.query); + + const query = { + $and: [notification.query, { + $or: [{ + 'token.apn': { + $exists: true, + }, + }, { + 'token.gcm': { + $exists: true, + }, + }], + }], + }; + + appTokensCollection.find(query).forEach((app) => { + logger.debug('send to token', app.token); + + if (app.token.apn) { + notification.topic = app.appName; + return sendGatewayPush(gateway, 'apn', app.token.apn, notification, options); + } + + if (app.token.gcm) { + return sendGatewayPush(gateway, 'gcm', app.token.gcm, notification, options); + } + }); + } +}; + +const serverSend = function(notification, options) { + if (options.gateways) { + return serverSendGateway(notification, options); + } + + return serverSendNative(notification, options); +}; + +export const configure = function(options) { options = _.extend({ sendTimeout: 60000, // Timeout period for notification send }, options); diff --git a/app/statistics/server/lib/statistics.js b/app/statistics/server/lib/statistics.js index 331bb110dc789..141a79b9e49f2 100644 --- a/app/statistics/server/lib/statistics.js +++ b/app/statistics/server/lib/statistics.js @@ -4,7 +4,7 @@ import _ from 'underscore'; import { Meteor } from 'meteor/meteor'; import { InstanceStatus } from 'meteor/konecty:multiple-instances-status'; -import { notificationsCollection } from '../../../push/server/push'; +import { notificationsCollection } from '../../../push/server'; import { Sessions, Settings, diff --git a/server/lib/pushConfig.js b/server/lib/pushConfig.js index 52fab8d894d13..14cc8324a38dc 100644 --- a/server/lib/pushConfig.js +++ b/server/lib/pushConfig.js @@ -1,13 +1,10 @@ import { Meteor } from 'meteor/meteor'; -import { HTTP } from 'meteor/http'; import { TAPi18n } from 'meteor/rocketchat:tap-i18n'; -import { Push } from 'meteor/rocketchat:push'; -import { SystemLogger } from '../../app/logger'; import { getWorkspaceAccessToken } from '../../app/cloud/server'; import { hasRole } from '../../app/authorization'; import { settings } from '../../app/settings'; -import { appTokensCollection } from '../../app/push/server/push'; +import { appTokensCollection, send, configure } from '../../app/push/server'; Meteor.methods({ @@ -30,7 +27,7 @@ Meteor.methods({ }); } - if (Push.enabled !== true) { + if (settings.get('Push_enable') !== true) { throw new Meteor.Error('error-push-disabled', 'Push is disabled', { method: 'push_test', }); @@ -60,7 +57,7 @@ Meteor.methods({ }); } - Push.send({ + send({ from: 'push', title: `@${ user.username }`, text: TAPi18n.__('This_is_a_push_test_messsage'), @@ -80,66 +77,8 @@ Meteor.methods({ }, }); -function sendPush(gateway, service, token, options, tries = 0) { - options.uniqueId = settings.get('uniqueID'); - - const data = { - data: { - token, - options, - }, - headers: {}, - }; - - const workspaceAccesstoken = getWorkspaceAccessToken(); - if (token) { - data.headers.Authorization = `Bearer ${ workspaceAccesstoken }`; - } - - return HTTP.post(`${ gateway }/push/${ service }/send`, data, function(error, response) { - if (response && response.statusCode === 406) { - console.log('removing push token', token); - appTokensCollection.remove({ - $or: [{ - 'token.apn': token, - }, { - 'token.gcm': token, - }], - }); - return; - } - - if (!error) { - return; - } - - SystemLogger.error(`Error sending push to gateway (${ tries } try) ->`, error); - - if (tries <= 6) { - const milli = Math.pow(10, tries + 2); - - SystemLogger.log('Trying sending push to gateway again in', milli, 'milliseconds'); - - return Meteor.setTimeout(function() { - return sendPush(gateway, service, token, options, tries + 1); - }, milli); - } - }); -} - function configurePush() { - if (settings.get('Push_debug')) { - Push.debug = true; - console.log('Push: configuring...'); - } - if (settings.get('Push_enable') === true) { - Push.allow({ - send(userId/* , notification*/) { - return hasRole(userId, 'admin'); - }, - }); - let apn; let gcm; @@ -173,65 +112,18 @@ function configurePush() { } } - Push.Configure({ + configure({ apn, gcm, production: settings.get('Push_production'), sendInterval: settings.get('Push_send_interval'), sendBatchSize: settings.get('Push_send_batch_size'), + gateways: settings.get('Push_enable_gateway') === true ? settings.get('Push_gateway').split('\n') : undefined, + uniqueId: settings.get('uniqueID'), + getAuthorization() { + return `Bearer ${ getWorkspaceAccessToken() }`; + }, }); - - if (settings.get('Push_enable_gateway') === true) { - Push.serverSend = function(options = { badge: 0 }) { - const gateways = settings.get('Push_gateway').split('\n'); - - for (const gateway of gateways) { - if (options.from !== String(options.from)) { - throw new Error('Push.send: option "from" not a string'); - } - if (options.title !== String(options.title)) { - throw new Error('Push.send: option "title" not a string'); - } - if (options.text !== String(options.text)) { - throw new Error('Push.send: option "text" not a string'); - } - if (settings.get('Push_debug')) { - console.log(`Push: send message "${ options.title }" via query`, options.query); - } - - const query = { - $and: [options.query, { - $or: [{ - 'token.apn': { - $exists: true, - }, - }, { - 'token.gcm': { - $exists: true, - }, - }], - }], - }; - - appTokensCollection.find(query).forEach((app) => { - if (settings.get('Push_debug')) { - console.log('Push: send to token', app.token); - } - - if (app.token.apn) { - options.topic = app.appName; - return sendPush(gateway, 'apn', app.token.apn, options); - } - - if (app.token.gcm) { - return sendPush(gateway, 'gcm', app.token.gcm, options); - } - }); - } - }; - } - - Push.enabled = true; } } diff --git a/server/startup/migrations/v181.js b/server/startup/migrations/v181.js index d0379ecf72caf..8f285d7293842 100644 --- a/server/startup/migrations/v181.js +++ b/server/startup/migrations/v181.js @@ -1,4 +1,4 @@ -import { notificationsCollection } from '../../../app/push/server/push'; +import { notificationsCollection } from '../../../app/push/server'; import { Migrations } from '../../../app/migrations/server'; import { Settings } from '../../../app/models/server'; From f7e9bc07f3b4e18115c88c731bb5e1f99c3dcafc Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 16 Apr 2020 22:35:09 -0300 Subject: [PATCH 04/40] Remove unused methods --- app/push/server/methods.js | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/app/push/server/methods.js b/app/push/server/methods.js index beb7604d03321..ac25aeacde2d5 100644 --- a/app/push/server/methods.js +++ b/app/push/server/methods.js @@ -101,49 +101,13 @@ Meteor.methods({ // Return the doc we want to use return doc; }, + // Deprecated 'raix:push-setuser'(id) { check(id, String); logger.debug(`Settings userId "${ this.userId }" for app:`, id); - // We update the appCollection id setting the Meteor.userId const found = appTokensCollection.update({ _id: id }, { $set: { userId: this.userId } }); - // Note that the app id might not exist because no token is set yet. - // We do create the new app id for the user since we might store additional - // metadata for the app / user - - // If id not found then create it? - // We dont, its better to wait until the user wants to - // store metadata or token - We could end up with unused data in the - // collection at every app re-install / update - // - // The user could store some metadata in appCollectin but only if they - // have created the app and provided a token. - // If not the metadata should be set via ground:db - - return !!found; - }, - 'raix:push-metadata'(data) { - check(data, { - id: String, - metadata: Object, - }); - - // Set the metadata - const found = appTokensCollection.update({ _id: data.id }, { $set: { metadata: data.metadata } }); - - return !!found; - }, - 'raix:push-enable'(data) { - check(data, { - id: String, - enabled: Boolean, - }); - - logger.debug(`Setting enabled to "${ data.enabled }" for app:`, data.id); - - const found = appTokensCollection.update({ _id: data.id }, { $set: { enabled: data.enabled } }); - return !!found; }, }); From d239f8ee124893529ed8f055583e0051d46c9729 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 16 Apr 2020 23:06:03 -0300 Subject: [PATCH 05/40] Remove dependecy of push package --- .meteor/packages | 2 -- 1 file changed, 2 deletions(-) diff --git a/.meteor/packages b/.meteor/packages index 32a7650706e9e..84ee48c2bc9e0 100644 --- a/.meteor/packages +++ b/.meteor/packages @@ -66,7 +66,6 @@ nooitaf:colors ostrio:cookies pauli:accounts-linkedin raix:handlebar-helpers -rocketchat:push raix:ui-dropped-event rocketchat:tap-i18n @@ -87,7 +86,6 @@ matb33:collection-hooks meteorhacks:inject-initial oauth@1.2.8 oauth2@1.2.1 -raix:eventemitter routepolicy@1.1.0 sha@1.0.9 templating From 4947f7b050318f9257898e1dfaffcebe9536da8b Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 16 Apr 2020 23:28:19 -0300 Subject: [PATCH 06/40] Convert push methods to class --- .../server/lib/PushNotification.js | 4 +- app/push/server/index.js | 4 +- app/push/server/push.js | 892 +++++++++--------- server/lib/pushConfig.js | 10 +- 4 files changed, 455 insertions(+), 455 deletions(-) diff --git a/app/push-notifications/server/lib/PushNotification.js b/app/push-notifications/server/lib/PushNotification.js index fce736ccdaa70..7a91478698eb8 100644 --- a/app/push-notifications/server/lib/PushNotification.js +++ b/app/push-notifications/server/lib/PushNotification.js @@ -1,4 +1,4 @@ -import { send } from '../../../push/server'; +import { Push } from '../../../push/server'; import { settings } from '../../../settings'; import { metrics } from '../../../metrics'; import { RocketChatAssets } from '../../../assets'; @@ -52,7 +52,7 @@ export class PushNotification { } metrics.notificationsSent.inc({ notification_type: 'mobile' }); - return send(config); + return Push.send(config); } } diff --git a/app/push/server/index.js b/app/push/server/index.js index ae97ba5ec9dab..35f25a9251d96 100644 --- a/app/push/server/index.js +++ b/app/push/server/index.js @@ -1 +1,3 @@ -export { send, appTokensCollection, notificationsCollection, configure } from './push'; +import './methods'; + +export { Push, appTokensCollection, notificationsCollection } from './push'; diff --git a/app/push/server/push.js b/app/push/server/push.js index bce84fc832e2a..a4268fa723f12 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -1,4 +1,3 @@ -/* eslint-disable new-cap */ import { Meteor } from 'meteor/meteor'; import { Match, check } from 'meteor/check'; import { Mongo } from 'meteor/mongo'; @@ -16,116 +15,298 @@ appTokensCollection._ensureIndex({ userId: 1 }); let isConfigured = false; -const sendWorker = function(task, interval) { - logger.debug(`Send worker started, using interval: ${ interval }`); +export class PushClass { + options = {} - return Meteor.setInterval(function() { - // xxx: add exponential backoff on error - try { - task(); - } catch (error) { - logger.debug(`Error while sending: ${ error.message }`); + configure(options) { + this.options = Object.assign({ + sendTimeout: 60000, // Timeout period for notification send + }, options); + // https://npmjs.org/package/apn + + // After requesting the certificate from Apple, export your private key as + // a .p12 file anddownload the .cer file from the iOS Provisioning Portal. + + // gateway.push.apple.com, port 2195 + // gateway.sandbox.push.apple.com, port 2195 + + // Now, in the directory containing cert.cer and key.p12 execute the + // following commands to generate your .pem files: + // $ openssl x509 -in cert.cer -inform DER -outform PEM -out cert.pem + // $ openssl pkcs12 -in key.p12 -out key.pem -nodes + + // Block multiple calls + if (isConfigured) { + throw new Error('Configure should not be called more than once!'); } - }, interval); -}; - -const _replaceToken = function(currentToken, newToken) { - appTokensCollection.update({ token: currentToken }, { $set: { token: newToken } }, { multi: true }); -}; - -const _removeToken = function(token) { - appTokensCollection.update({ token }, { $unset: { token: true } }, { multi: true }); -}; - -// Universal send function -const _querySend = function(query, notification, options) { - const countApn = []; - const countGcm = []; - - appTokensCollection.find(query).forEach(function(app) { - logger.debug('send to token', app.token); - - if (app.token.apn) { - countApn.push(app._id); - // Send to APN - if (options.apn) { - sendAPN(app.token.apn, notification); - } - } else if (app.token.gcm) { - countGcm.push(app._id); - - // Send to GCM - // We do support multiple here - so we should construct an array - // and send it bulk - Investigate limit count of id's - if (options.gcm && options.gcm.apiKey) { - sendGCM({ userTokens: app.token.gcm, notification, _replaceToken, _removeToken, options }); - } + + isConfigured = true; + + logger.debug('Configure', this.options); + + if (this.options.apn) { + initAPN({ options: this.options, _removeToken: this._removeToken }); + } // EO ios notification + + // This interval will allow only one notification to be sent at a time, it + // will check for new notifications at every `options.sendInterval` + // (default interval is 15000 ms) + // + // It looks in notifications collection to see if theres any pending + // notifications, if so it will try to reserve the pending notification. + // If successfully reserved the send is started. + // + // If notification.query is type string, it's assumed to be a json string + // version of the query selector. Making it able to carry `$` properties in + // the mongo collection. + // + // Pr. default notifications are removed from the collection after send have + // completed. Setting `options.keepNotifications` will update and keep the + // notification eg. if needed for historical reasons. + // + // After the send have completed a "send" event will be emitted with a + // status object containing notification id and the send result object. + // + let isSendingNotification = false; + + if (this.options.sendInterval !== null) { + // This will require index since we sort notifications by createdAt + notificationsCollection._ensureIndex({ createdAt: 1 }); + notificationsCollection._ensureIndex({ sent: 1 }); + notificationsCollection._ensureIndex({ sending: 1 }); + notificationsCollection._ensureIndex({ delayUntil: 1 }); + + const sendNotification = (notification) => { + // Reserve notification + const now = +new Date(); + const timeoutAt = now + this.options.sendTimeout; + const reserved = notificationsCollection.update({ + _id: notification._id, + sent: false, // xxx: need to make sure this is set on create + sending: { $lt: now }, + }, + { + $set: { + sending: timeoutAt, + }, + }); + + // Make sure we only handle notifications reserved by this + // instance + if (reserved) { + // Check if query is set and is type String + if (notification.query && notification.query === `${ notification.query }`) { + try { + // The query is in string json format - we need to parse it + notification.query = JSON.parse(notification.query); + } catch (err) { + // Did the user tamper with this?? + throw new Error(`Error while parsing query string, Error: ${ err.message }`); + } + } + + // Send the notification + const result = this.serverSend(notification, this.options); + + if (!this.options.keepNotifications) { + // Pr. Default we will remove notifications + notificationsCollection.remove({ _id: notification._id }); + } else { + // Update the notification + notificationsCollection.update({ _id: notification._id }, { + $set: { + // Mark as sent + sent: true, + // Set the sent date + sentAt: new Date(), + // Count + count: result, + // Not being sent anymore + sending: 0, + }, + }); + } + + // Emit the send + // self.emit('send', { notification: notification._id, result }); + } // Else could not reserve + }; // EO sendNotification + + this.sendWorker(() => { + if (isSendingNotification) { + return; + } + + try { + // Set send fence + isSendingNotification = true; + + // var countSent = 0; + const batchSize = this.options.sendBatchSize || 1; + + const now = +new Date(); + + // Find notifications that are not being or already sent + const pendingNotifications = notificationsCollection.find({ $and: [ + // Message is not sent + { sent: false }, + // And not being sent by other instances + { sending: { $lt: now } }, + // And not queued for future + { $or: [ + { delayUntil: { $exists: false } }, + { delayUntil: { $lte: new Date() } }, + ], + }, + ] }, { + // Sort by created date + sort: { createdAt: 1 }, + limit: batchSize, + }); + + pendingNotifications.forEach((notification) => { + try { + sendNotification(notification); + } catch (error) { + logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); + } + }); // EO forEach + } finally { + // Remove the send fence + isSendingNotification = false; + } + }, this.options.sendInterval || 15000); // Default every 15th sec } else { - throw new Error('send got a faulty query'); + logger.debug('Send server is disabled'); } - }); + } - if (LoggerManager.logLevel === 2) { - logger.debug(`Sent message "${ notification.title }" to ${ countApn.length } ios apps ${ countGcm.length } android apps`); + sendWorker(task, interval) { + logger.debug(`Send worker started, using interval: ${ interval }`); - // Add some verbosity about the send result, making sure the developer - // understands what just happened. - if (!countApn.length && !countGcm.length) { - if (appTokensCollection.find().count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" is empty - No clients have registered on the server yet...'); + return Meteor.setInterval(function() { + // xxx: add exponential backoff on error + try { + task(); + } catch (error) { + logger.debug(`Error while sending: ${ error.message }`); } - } else if (!countApn.length) { - if (appTokensCollection.find({ 'token.apn': { $exists: true } }).count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" - No APN clients have registered on the server yet...'); + }, interval); + } + + _replaceToken(currentToken, newToken) { + appTokensCollection.update({ token: currentToken }, { $set: { token: newToken } }, { multi: true }); + } + + _removeToken(token) { + appTokensCollection.update({ token }, { $unset: { token: true } }, { multi: true }); + } + + // Universal send function + _querySend(query, notification) { + const countApn = []; + const countGcm = []; + + appTokensCollection.find(query).forEach((app) => { + logger.debug('send to token', app.token); + + if (app.token.apn) { + countApn.push(app._id); + // Send to APN + if (this.options.apn) { + sendAPN(app.token.apn, notification); + } + } else if (app.token.gcm) { + countGcm.push(app._id); + + // Send to GCM + // We do support multiple here - so we should construct an array + // and send it bulk - Investigate limit count of id's + if (this.options.gcm && this.options.gcm.apiKey) { + sendGCM({ userTokens: app.token.gcm, notification, _replaceToken: this._replaceToken, _removeToken: this._removeToken, options: this.options }); + } + } else { + throw new Error('send got a faulty query'); } - } else if (!countGcm.length) { - if (appTokensCollection.find({ 'token.gcm': { $exists: true } }).count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" - No GCM clients have registered on the server yet...'); + }); + + if (LoggerManager.logLevel === 2) { + logger.debug(`Sent message "${ notification.title }" to ${ countApn.length } ios apps ${ countGcm.length } android apps`); + + // Add some verbosity about the send result, making sure the developer + // understands what just happened. + if (!countApn.length && !countGcm.length) { + if (appTokensCollection.find().count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" is empty - No clients have registered on the server yet...'); + } + } else if (!countApn.length) { + if (appTokensCollection.find({ 'token.apn': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No APN clients have registered on the server yet...'); + } + } else if (!countGcm.length) { + if (appTokensCollection.find({ 'token.gcm': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No GCM clients have registered on the server yet...'); + } } } - } - return { - apn: countApn, - gcm: countGcm, - }; -}; + return { + apn: countApn, + gcm: countGcm, + }; + } -const serverSendNative = function(notification, options) { - notification = notification || { badge: 0 }; - let query; + serverSendNative(notification) { + notification = notification || { badge: 0 }; + let query; - // Check basic options - if (notification.from !== `${ notification.from }`) { - throw new Error('send: option "from" not a string'); - } + // Check basic options + if (notification.from !== `${ notification.from }`) { + throw new Error('send: option "from" not a string'); + } - if (notification.title !== `${ notification.title }`) { - throw new Error('send: option "title" not a string'); - } + if (notification.title !== `${ notification.title }`) { + throw new Error('send: option "title" not a string'); + } - if (notification.text !== `${ notification.text }`) { - throw new Error('send: option "text" not a string'); - } + if (notification.text !== `${ notification.text }`) { + throw new Error('send: option "text" not a string'); + } - if (notification.token || notification.tokens) { - // The user set one token or array of tokens - const tokenList = notification.token ? [notification.token] : notification.tokens; + if (notification.token || notification.tokens) { + // The user set one token or array of tokens + const tokenList = notification.token ? [notification.token] : notification.tokens; - logger.debug(`Send message "${ notification.title }" via token(s)`, tokenList); + logger.debug(`Send message "${ notification.title }" via token(s)`, tokenList); - query = { - $or: [ - // XXX: Test this query: can we hand in a list of push tokens? - { $and: [ - { token: { $in: tokenList } }, - // And is not disabled - { enabled: { $ne: false } }, + query = { + $or: [ + // XXX: Test this query: can we hand in a list of push tokens? + { $and: [ + { token: { $in: tokenList } }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }, + // XXX: Test this query: does this work on app id? + { $and: [ + { _id: { $in: tokenList } }, // one of the app ids + { $or: [ + { 'token.apn': { $exists: true } }, // got apn token + { 'token.gcm': { $exists: true } }, // got gcm token + ] }, + // And is not disabled + { enabled: { $ne: false } }, + ], + }, ], - }, - // XXX: Test this query: does this work on app id? - { $and: [ - { _id: { $in: tokenList } }, // one of the app ids + }; + } else if (notification.query) { + logger.debug(`Send message "${ notification.title }" via query`, notification.query); + + query = { + $and: [ + notification.query, // query object { $or: [ { 'token.apn': { $exists: true } }, // got apn token { 'token.gcm': { $exists: true } }, // got gcm token @@ -133,394 +314,215 @@ const serverSendNative = function(notification, options) { // And is not disabled { enabled: { $ne: false } }, ], - }, - ], - }; - } else if (notification.query) { - logger.debug(`Send message "${ notification.title }" via query`, notification.query); - - query = { - $and: [ - notification.query, // query object - { $or: [ - { 'token.apn': { $exists: true } }, // got apn token - { 'token.gcm': { $exists: true } }, // got gcm token - ] }, - // And is not disabled - { enabled: { $ne: false } }, - ], - }; - } - - if (query) { - // Convert to querySend and return status - return _querySend(query, notification, options); - } - throw new Error('send: please set option "token"/"tokens" or "query"'); -}; - -const sendGatewayPush = (gateway, service, token, notification, options, tries = 0) => { - notification.uniqueId = options.uniqueId; - - const data = { - data: { - token, - options: notification, - }, - headers: {}, - }; - - if (token && options.getAuthorization) { - data.headers.Authorization = options.getAuthorization(); - } - - return HTTP.post(`${ gateway }/push/${ service }/send`, data, function(error, response) { - if (response && response.statusCode === 406) { - console.log('removing push token', token); - appTokensCollection.remove({ - $or: [{ - 'token.apn': token, - }, { - 'token.gcm': token, - }], - }); - return; + }; } - if (!error) { - return; + if (query) { + // Convert to querySend and return status + return this._querySend(query, notification); } + throw new Error('send: please set option "token"/"tokens" or "query"'); + } - logger.error(`Error sending push to gateway (${ tries } try) ->`, error); - - if (tries <= 6) { - const ms = Math.pow(10, tries + 2); + sendGatewayPush(gateway, service, token, notification, tries = 0) { + notification.uniqueId = this.options.uniqueId; - logger.log('Trying sending push to gateway again in', ms, 'milliseconds'); + const data = { + data: { + token, + options: notification, + }, + headers: {}, + }; - return Meteor.setTimeout(function() { - return sendGatewayPush(gateway, service, token, notification, options, tries + 1); - }, ms); + if (token && this.options.getAuthorization) { + data.headers.Authorization = this.options.getAuthorization(); } - }); -}; -const serverSendGateway = function(notification = { badge: 0 }, options) { - for (const gateway of options.gateways) { - if (notification.from !== String(notification.from)) { - throw new Error('Push.send: option "from" not a string'); - } - if (notification.title !== String(notification.title)) { - throw new Error('Push.send: option "title" not a string'); - } - if (notification.text !== String(notification.text)) { - throw new Error('Push.send: option "text" not a string'); - } + return HTTP.post(`${ gateway }/push/${ service }/send`, data, (error, response) => { + if (response && response.statusCode === 406) { + console.log('removing push token', token); + appTokensCollection.remove({ + $or: [{ + 'token.apn': token, + }, { + 'token.gcm': token, + }], + }); + return; + } - logger.debug(`send message "${ notification.title }" via query`, notification.query); + if (!error) { + return; + } - const query = { - $and: [notification.query, { - $or: [{ - 'token.apn': { - $exists: true, - }, - }, { - 'token.gcm': { - $exists: true, - }, - }], - }], - }; + logger.error(`Error sending push to gateway (${ tries } try) ->`, error); - appTokensCollection.find(query).forEach((app) => { - logger.debug('send to token', app.token); + if (tries <= 6) { + const ms = Math.pow(10, tries + 2); - if (app.token.apn) { - notification.topic = app.appName; - return sendGatewayPush(gateway, 'apn', app.token.apn, notification, options); - } + logger.log('Trying sending push to gateway again in', ms, 'milliseconds'); - if (app.token.gcm) { - return sendGatewayPush(gateway, 'gcm', app.token.gcm, notification, options); + return Meteor.setTimeout(() => this.sendGatewayPush(gateway, service, token, notification, tries + 1), ms); } }); } -}; - -const serverSend = function(notification, options) { - if (options.gateways) { - return serverSendGateway(notification, options); - } - - return serverSendNative(notification, options); -}; -export const configure = function(options) { - options = _.extend({ - sendTimeout: 60000, // Timeout period for notification send - }, options); - // https://npmjs.org/package/apn + serverSendGateway(notification = { badge: 0 }) { + for (const gateway of this.options.gateways) { + if (notification.from !== String(notification.from)) { + throw new Error('Push.send: option "from" not a string'); + } + if (notification.title !== String(notification.title)) { + throw new Error('Push.send: option "title" not a string'); + } + if (notification.text !== String(notification.text)) { + throw new Error('Push.send: option "text" not a string'); + } - // After requesting the certificate from Apple, export your private key as - // a .p12 file anddownload the .cer file from the iOS Provisioning Portal. + logger.debug(`send message "${ notification.title }" via query`, notification.query); - // gateway.push.apple.com, port 2195 - // gateway.sandbox.push.apple.com, port 2195 + const query = { + $and: [notification.query, { + $or: [{ + 'token.apn': { + $exists: true, + }, + }, { + 'token.gcm': { + $exists: true, + }, + }], + }], + }; - // Now, in the directory containing cert.cer and key.p12 execute the - // following commands to generate your .pem files: - // $ openssl x509 -in cert.cer -inform DER -outform PEM -out cert.pem - // $ openssl pkcs12 -in key.p12 -out key.pem -nodes + appTokensCollection.find(query).forEach((app) => { + logger.debug('send to token', app.token); - // Block multiple calls - if (isConfigured) { - throw new Error('Configure should not be called more than once!'); - } + if (app.token.apn) { + notification.topic = app.appName; + return this.sendGatewayPush(gateway, 'apn', app.token.apn, notification); + } - isConfigured = true; - - logger.debug('Configure', options); - - if (options.apn) { - initAPN({ options, _removeToken }); - } // EO ios notification - - // This interval will allow only one notification to be sent at a time, it - // will check for new notifications at every `options.sendInterval` - // (default interval is 15000 ms) - // - // It looks in notifications collection to see if theres any pending - // notifications, if so it will try to reserve the pending notification. - // If successfully reserved the send is started. - // - // If notification.query is type string, it's assumed to be a json string - // version of the query selector. Making it able to carry `$` properties in - // the mongo collection. - // - // Pr. default notifications are removed from the collection after send have - // completed. Setting `options.keepNotifications` will update and keep the - // notification eg. if needed for historical reasons. - // - // After the send have completed a "send" event will be emitted with a - // status object containing notification id and the send result object. - // - let isSendingNotification = false; - - if (options.sendInterval !== null) { - // This will require index since we sort notifications by createdAt - notificationsCollection._ensureIndex({ createdAt: 1 }); - notificationsCollection._ensureIndex({ sent: 1 }); - notificationsCollection._ensureIndex({ sending: 1 }); - notificationsCollection._ensureIndex({ delayUntil: 1 }); - - const sendNotification = function(notification) { - // Reserve notification - const now = +new Date(); - const timeoutAt = now + options.sendTimeout; - const reserved = notificationsCollection.update({ - _id: notification._id, - sent: false, // xxx: need to make sure this is set on create - sending: { $lt: now }, - }, - { - $set: { - sending: timeoutAt, - }, + if (app.token.gcm) { + return this.sendGatewayPush(gateway, 'gcm', app.token.gcm, notification); + } }); + } + } - // Make sure we only handle notifications reserved by this - // instance - if (reserved) { - // Check if query is set and is type String - if (notification.query && notification.query === `${ notification.query }`) { - try { - // The query is in string json format - we need to parse it - notification.query = JSON.parse(notification.query); - } catch (err) { - // Did the user tamper with this?? - throw new Error(`Error while parsing query string, Error: ${ err.message }`); - } - } + serverSend(notification) { + if (this.options.gateways) { + return this.serverSendGateway(notification); + } - // Send the notification - const result = serverSend(notification, options); - - if (!options.keepNotifications) { - // Pr. Default we will remove notifications - notificationsCollection.remove({ _id: notification._id }); - } else { - // Update the notification - notificationsCollection.update({ _id: notification._id }, { - $set: { - // Mark as sent - sent: true, - // Set the sent date - sentAt: new Date(), - // Count - count: result, - // Not being sent anymore - sending: 0, - }, - }); - } + return this.serverSendNative(notification); + } - // Emit the send - // self.emit('send', { notification: notification._id, result }); - } // Else could not reserve - }; // EO sendNotification + // This is a general function to validate that the data added to notifications + // is in the correct format. If not this function will throw errors + _validateDocument(notification) { + // Check the general notification + check(notification, { + from: String, + title: String, + text: String, + sent: Match.Optional(Boolean), + sending: Match.Optional(Match.Integer), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + contentAvailable: Match.Optional(Match.Integer), + apn: Match.Optional({ + from: Match.Optional(String), + title: Match.Optional(String), + text: Match.Optional(String), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + category: Match.Optional(String), + }), + gcm: Match.Optional({ + from: Match.Optional(String), + title: Match.Optional(String), + text: Match.Optional(String), + image: Match.Optional(String), + style: Match.Optional(String), + summaryText: Match.Optional(String), + picture: Match.Optional(String), + badge: Match.Optional(Match.Integer), + sound: Match.Optional(String), + notId: Match.Optional(Match.Integer), + }), + query: Match.Optional(String), + token: Match.Optional(_matchToken), + tokens: Match.Optional([_matchToken]), + payload: Match.Optional(Object), + delayUntil: Match.Optional(Date), + createdAt: Date, + createdBy: Match.OneOf(String, null), + }); - sendWorker(function() { - if (isSendingNotification) { - return; - } + // Make sure a token selector or query have been set + if (!notification.token && !notification.tokens && !notification.query) { + throw new Error('No token selector or query found'); + } - try { - // Set send fence - isSendingNotification = true; + // If tokens array is set it should not be empty + if (notification.tokens && !notification.tokens.length) { + throw new Error('No tokens in array'); + } + } - // var countSent = 0; - const batchSize = options.sendBatchSize || 1; + send(options) { + // If on the client we set the user id - on the server we need an option + // set or we default to "" as the creator of the notification + // If current user not set see if we can set it to the logged in user + // this will only run on the client if Meteor.userId is available + const currentUser = options.createdBy || ''; - const now = +new Date(); + // Rig the notification object + const notification = Object.assign({ + createdAt: new Date(), + createdBy: currentUser, + }, _.pick(options, 'from', 'title', 'text')); - // Find notifications that are not being or already sent - const pendingNotifications = notificationsCollection.find({ $and: [ - // Message is not sent - { sent: false }, - // And not being sent by other instances - { sending: { $lt: now } }, - // And not queued for future - { $or: [ - { delayUntil: { $exists: false } }, - { delayUntil: { $lte: new Date() } }, - ], - }, - ] }, { - // Sort by created date - sort: { createdAt: 1 }, - limit: batchSize, - }); + // Add extra + Object.assign(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil')); - pendingNotifications.forEach(function(notification) { - try { - sendNotification(notification); - } catch (error) { - logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); - } - }); // EO forEach - } finally { - // Remove the send fence - isSendingNotification = false; - } - }, options.sendInterval || 15000); // Default every 15th sec - } else { - logger.debug('Send server is disabled'); - } -}; - - -// This is a general function to validate that the data added to notifications -// is in the correct format. If not this function will throw errors -const _validateDocument = function(notification) { - // Check the general notification - check(notification, { - from: String, - title: String, - text: String, - sent: Match.Optional(Boolean), - sending: Match.Optional(Match.Integer), - badge: Match.Optional(Match.Integer), - sound: Match.Optional(String), - notId: Match.Optional(Match.Integer), - contentAvailable: Match.Optional(Match.Integer), - apn: Match.Optional({ - from: Match.Optional(String), - title: Match.Optional(String), - text: Match.Optional(String), - badge: Match.Optional(Match.Integer), - sound: Match.Optional(String), - notId: Match.Optional(Match.Integer), - category: Match.Optional(String), - }), - gcm: Match.Optional({ - from: Match.Optional(String), - title: Match.Optional(String), - text: Match.Optional(String), - image: Match.Optional(String), - style: Match.Optional(String), - summaryText: Match.Optional(String), - picture: Match.Optional(String), - badge: Match.Optional(Match.Integer), - sound: Match.Optional(String), - notId: Match.Optional(Match.Integer), - }), - query: Match.Optional(String), - token: Match.Optional(_matchToken), - tokens: Match.Optional([_matchToken]), - payload: Match.Optional(Object), - delayUntil: Match.Optional(Date), - createdAt: Date, - createdBy: Match.OneOf(String, null), - }); - - // Make sure a token selector or query have been set - if (!notification.token && !notification.tokens && !notification.query) { - throw new Error('No token selector or query found'); - } + if (Match.test(options.apn, Object)) { + notification.apn = _.pick(options.apn, 'from', 'title', 'text', 'badge', 'sound', 'notId', 'category'); + } - // If tokens array is set it should not be empty - if (notification.tokens && !notification.tokens.length) { - throw new Error('No tokens in array'); - } -}; - -export const send = function(options) { - // If on the client we set the user id - on the server we need an option - // set or we default to "" as the creator of the notification - // If current user not set see if we can set it to the logged in user - // this will only run on the client if Meteor.userId is available - const currentUser = options.createdBy || ''; - - // Rig the notification object - const notification = _.extend({ - createdAt: new Date(), - createdBy: currentUser, - }, _.pick(options, 'from', 'title', 'text')); - - // Add extra - _.extend(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil')); - - if (Match.test(options.apn, Object)) { - notification.apn = _.pick(options.apn, 'from', 'title', 'text', 'badge', 'sound', 'notId', 'category'); - } + if (Match.test(options.gcm, Object)) { + notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId'); + } - if (Match.test(options.gcm, Object)) { - notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId'); - } + // Set one token selector, this can be token, array of tokens or query + if (options.query) { + // Set query to the json string version fixing #43 and #39 + notification.query = JSON.stringify(options.query); + } else if (options.token) { + // Set token + notification.token = options.token; + } else if (options.tokens) { + // Set tokens + notification.tokens = options.tokens; + } + // console.log(options); + if (typeof options.contentAvailable !== 'undefined') { + notification.contentAvailable = options.contentAvailable; + } - // Set one token selector, this can be token, array of tokens or query - if (options.query) { - // Set query to the json string version fixing #43 and #39 - notification.query = JSON.stringify(options.query); - } else if (options.token) { - // Set token - notification.token = options.token; - } else if (options.tokens) { - // Set tokens - notification.tokens = options.tokens; - } - // console.log(options); - if (typeof options.contentAvailable !== 'undefined') { - notification.contentAvailable = options.contentAvailable; - } + notification.sent = false; + notification.sending = 0; - notification.sent = false; - notification.sending = 0; + // Validate the notification + this._validateDocument(notification); - // Validate the notification - _validateDocument(notification); + // Try to add the notification to send, we return an id to keep track + return notificationsCollection.insert(notification); + } +} - // Try to add the notification to send, we return an id to keep track - return notificationsCollection.insert(notification); -}; +export const Push = new PushClass(); diff --git a/server/lib/pushConfig.js b/server/lib/pushConfig.js index 14cc8324a38dc..b85bbea481ba3 100644 --- a/server/lib/pushConfig.js +++ b/server/lib/pushConfig.js @@ -4,14 +4,10 @@ import { TAPi18n } from 'meteor/rocketchat:tap-i18n'; import { getWorkspaceAccessToken } from '../../app/cloud/server'; import { hasRole } from '../../app/authorization'; import { settings } from '../../app/settings'; -import { appTokensCollection, send, configure } from '../../app/push/server'; +import { appTokensCollection, Push } from '../../app/push/server'; Meteor.methods({ - // log() { - // return console.log(...arguments); - // }, - push_test() { const user = Meteor.user(); @@ -57,7 +53,7 @@ Meteor.methods({ }); } - send({ + Push.send({ from: 'push', title: `@${ user.username }`, text: TAPi18n.__('This_is_a_push_test_messsage'), @@ -112,7 +108,7 @@ function configurePush() { } } - configure({ + Push.configure({ apn, gcm, production: settings.get('Push_production'), From 5ae70d4c27e0cc3d65d853e68cdbf19499deaffc Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 16 Apr 2020 23:45:45 -0300 Subject: [PATCH 07/40] Improve GCM and APN codes --- app/push/server/apn.js | 31 +++++----------- app/push/server/gcm.js | 82 ++++++++++++----------------------------- app/push/server/push.js | 4 +- 3 files changed, 35 insertions(+), 82 deletions(-) diff --git a/app/push/server/apn.js b/app/push/server/apn.js index fe2440c162646..23eb5217a860d 100644 --- a/app/push/server/apn.js +++ b/app/push/server/apn.js @@ -1,20 +1,17 @@ import { Meteor } from 'meteor/meteor'; import { Match } from 'meteor/check'; import { EJSON } from 'meteor/ejson'; -import _ from 'underscore'; import apn from 'apn'; import { logger } from './logger'; let apnConnection; -export const sendAPN = function(userToken, notification) { +export const sendAPN = (userToken, notification) => { if (Match.test(notification.apn, Object)) { - notification = _.extend({}, notification, notification.apn); + notification = Object.assign({}, notification, notification.apn); } - // console.log('sendAPN', notification.from, userToken, notification.title, notification.text, - // notification.badge, notification.priority); const priority = notification.priority || notification.priority === 0 ? notification.priority : 10; const myDevice = new apn.Device(userToken); @@ -22,25 +19,20 @@ export const sendAPN = function(userToken, notification) { const note = new apn.Notification(); note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now. - if (typeof notification.badge !== 'undefined') { + if (notification.badge != null) { note.badge = notification.badge; } - if (typeof notification.sound !== 'undefined') { + if (notification.sound != null) { note.sound = notification.sound; } - // console.log(notification.contentAvailable); - // console.log("lala2"); - // console.log(notification); - if (typeof notification.contentAvailable !== 'undefined') { - // console.log("lala"); + if (notification.contentAvailable != null) { note.setContentAvailable(notification.contentAvailable); - // console.log(note); } // adds category support for iOS8 custom actions as described here: // https://developer.apple.com/library/ios/documentation/NetworkingInternet/Conceptual/ // RemoteNotificationsPG/Chapters/IPhoneOSClientImp.html#//apple_ref/doc/uid/TP40008194-CH103-SW36 - if (typeof notification.category !== 'undefined') { + if (notification.category != null) { note.category = notification.category; } @@ -48,7 +40,7 @@ export const sendAPN = function(userToken, notification) { body: notification.text, }; - if (typeof notification.title !== 'undefined') { + if (notification.title != null) { note.alert.title = notification.title; } @@ -58,19 +50,16 @@ export const sendAPN = function(userToken, notification) { note.payload.messageFrom = notification.from; note.priority = priority; - // Store the token on the note so we can reference it if there was an error note.token = userToken; - // console.log('I:Send message to: ' + userToken + ' count=' + count); - apnConnection.pushNotification(note, myDevice); }; // Init feedback from apn server // This will help keep the appCollection up-to-date, it will help update // and remove token from appCollection. -export const initFeedback = function({ options, _removeToken }) { +export const initFeedback = ({ options, _removeToken }) => { // console.log('Init feedback'); const feedbackOptions = { batchFeedback: true, @@ -84,8 +73,8 @@ export const initFeedback = function({ options, _removeToken }) { }; const feedback = new apn.Feedback(feedbackOptions); - feedback.on('feedback', function(devices) { - devices.forEach(function(item) { + feedback.on('feedback', (devices) => { + devices.forEach((item) => { // Do something with item.device and item.time; // console.log('A:PUSH FEEDBACK ' + item.device + ' - ' + item.time); // The app is most likely removed from the device, we should diff --git a/app/push/server/gcm.js b/app/push/server/gcm.js index 3bbcf5845138c..820f52cf67b65 100644 --- a/app/push/server/gcm.js +++ b/app/push/server/gcm.js @@ -1,19 +1,17 @@ /* eslint-disable new-cap */ import { Match } from 'meteor/check'; import { EJSON } from 'meteor/ejson'; -import _ from 'underscore'; import gcm from 'node-gcm'; -import Fiber from 'fibers'; import { logger } from './logger'; export const sendGCM = function({ userTokens, notification, _replaceToken, _removeToken, options }) { if (Match.test(notification.gcm, Object)) { - notification = _.extend({}, notification, notification.gcm); + notification = Object.assign({}, notification, notification.gcm); } // Make sure userTokens are an array of strings - if (userTokens === `${ userTokens }`) { + if (Match.test(userTokens, String)) { userTokens = [userTokens]; } @@ -32,31 +30,30 @@ export const sendGCM = function({ userTokens, notification, _replaceToken, _remo data.message = notification.text; // Set image - if (typeof notification.image !== 'undefined') { + if (notification.image != null) { data.image = notification.image; } // Set extra details - if (typeof notification.badge !== 'undefined') { + if (notification.badge != null) { data.msgcnt = notification.badge; } - if (typeof notification.sound !== 'undefined') { + if (notification.sound != null) { data.soundname = notification.sound; } - if (typeof notification.notId !== 'undefined') { + if (notification.notId != null) { data.notId = notification.notId; } - if (typeof notification.style !== 'undefined') { + if (notification.style != null) { data.style = notification.style; } - if (typeof notification.summaryText !== 'undefined') { + if (notification.summaryText != null) { data.summaryText = notification.summaryText; } - if (typeof notification.picture !== 'undefined') { + if (notification.picture != null) { data.picture = notification.picture; } - // var message = new gcm.Message(); const message = new gcm.Message({ collapseKey: notification.from, // delayWhileIdle: true, @@ -68,20 +65,7 @@ export const sendGCM = function({ userTokens, notification, _replaceToken, _remo logger.debug(`Create GCM Sender using "${ options.gcm.apiKey }"`); const sender = new gcm.Sender(options.gcm.apiKey); - _.each(userTokens, function(value /* , key */) { - logger.debug(`A:Send message to: ${ value }`); - }); - - /* message.addData('title', title); - message.addData('message', text); - message.addData('msgcnt', '1'); - message.collapseKey = 'sitDrift'; - message.delayWhileIdle = true; - message.timeToLive = 3;*/ - - // /** - // * Parameters: message-literal, userTokens-array, No. of retries, callback-function - // */ + userTokens.forEach((value) => logger.debug(`A:Send message to: ${ value }`)); const userToken = userTokens.length === 1 ? userTokens[0] : null; @@ -98,43 +82,23 @@ export const sendGCM = function({ userTokens, notification, _replaceToken, _remo logger.debuglog(`ANDROID: Result of sender: ${ JSON.stringify(result) }`); - if (result.canonical_ids === 1 && userToken) { // jshint ignore:line + if (result.canonical_ids === 1 && userToken) { // This is an old device, token is replaced - Fiber(function(self) { - // Run in fiber - try { - self.callback(self.oldToken, self.newToken); - } catch (err) { - // - } - }).run({ - oldToken: { gcm: userToken }, - newToken: { gcm: result.results[0].registration_id }, // jshint ignore:line - callback: _replaceToken, - }); - // _replaceToken({ gcm: userToken }, { gcm: result.results[0].registration_id }); + try { + _replaceToken({ gcm: userToken }, { gcm: result.results[0].registration_id }); + } catch (err) { + logger.error('Error replacing token', err); + } } - // We cant send to that token - might not be registred + // We cant send to that token - might not be registered // ask the user to remove the token from the list if (result.failure !== 0 && userToken) { // This is an old device, token is replaced - Fiber(function(self) { - // Run in fiber - try { - self.callback(self.token); - } catch (err) { - // - } - }).run({ - token: { gcm: userToken }, - callback: _removeToken, - }); - // _replaceToken({ gcm: userToken }, { gcm: result.results[0].registration_id }); + try { + _removeToken({ gcm: userToken }); + } catch (err) { + logger.error('Error removing token', err); + } } }); - // /** Use the following line if you want to send the message without retries - // sender.sendNoRetry(message, userTokens, function (result) { - // console.log('ANDROID: ' + JSON.stringify(result)); - // }); - // **/ -}; // EO sendAndroid +}; diff --git a/app/push/server/push.js b/app/push/server/push.js index a4268fa723f12..9b1316f9e207d 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -195,11 +195,11 @@ export class PushClass { } _replaceToken(currentToken, newToken) { - appTokensCollection.update({ token: currentToken }, { $set: { token: newToken } }, { multi: true }); + appTokensCollection.rawCollection().updateMany({ token: currentToken }, { $set: { token: newToken } }); } _removeToken(token) { - appTokensCollection.update({ token }, { $unset: { token: true } }, { multi: true }); + appTokensCollection.rawCollection().updateMany({ token }, { $unset: { token: true } }); } // Universal send function From 16c0cd20d39ce7fae3f47a3dd6417cc69ff8aac6 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 17 Apr 2020 00:25:44 -0300 Subject: [PATCH 08/40] Update APN --- app/push/server/apn.js | 41 +++-------------------------------------- app/push/server/push.js | 1 + package-lock.json | 32 ++++++++++++++++++++++++-------- package.json | 2 +- 4 files changed, 29 insertions(+), 47 deletions(-) diff --git a/app/push/server/apn.js b/app/push/server/apn.js index 23eb5217a860d..351db7de28aca 100644 --- a/app/push/server/apn.js +++ b/app/push/server/apn.js @@ -14,8 +14,6 @@ export const sendAPN = (userToken, notification) => { const priority = notification.priority || notification.priority === 0 ? notification.priority : 10; - const myDevice = new apn.Device(userToken); - const note = new apn.Notification(); note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now. @@ -52,40 +50,9 @@ export const sendAPN = (userToken, notification) => { // Store the token on the note so we can reference it if there was an error note.token = userToken; + note.topic = notification.topic; - apnConnection.pushNotification(note, myDevice); -}; - -// Init feedback from apn server -// This will help keep the appCollection up-to-date, it will help update -// and remove token from appCollection. -export const initFeedback = ({ options, _removeToken }) => { - // console.log('Init feedback'); - const feedbackOptions = { - batchFeedback: true, - - // Time in SECONDS - interval: 5, - production: !options.apn.development, - cert: options.certData, - key: options.keyData, - passphrase: options.passphrase, - }; - - const feedback = new apn.Feedback(feedbackOptions); - feedback.on('feedback', (devices) => { - devices.forEach((item) => { - // Do something with item.device and item.time; - // console.log('A:PUSH FEEDBACK ' + item.device + ' - ' + item.time); - // The app is most likely removed from the device, we should - // remove the token - _removeToken({ - apn: item.device, - }); - }); - }); - - feedback.start(); + apnConnection.send(note, userToken); }; export const initAPN = ({ options, _removeToken }) => { @@ -141,7 +108,7 @@ export const initAPN = ({ options, _removeToken }) => { } // Rig apn connection - apnConnection = new apn.Connection(options.apn); + apnConnection = new apn.Provider(options.apn); // Listen to transmission errors - should handle the same way as feedback. apnConnection.on('transmissionError', Meteor.bindEnvironment(function(errCode, notification/* , recipient*/) { @@ -154,6 +121,4 @@ export const initAPN = ({ options, _removeToken }) => { }); } })); - - initFeedback({ options, _removeToken }); }; diff --git a/app/push/server/push.js b/app/push/server/push.js index 9b1316f9e207d..f926904a758d0 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -214,6 +214,7 @@ export class PushClass { countApn.push(app._id); // Send to APN if (this.options.apn) { + notification.topic = app.appName; sendAPN(app.token.apn, notification); } } else if (app.token.gcm) { diff --git a/package-lock.json b/package-lock.json index 85a093b274b90..a8973ad4a0a12 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7035,17 +7035,29 @@ "dev": true }, "apn": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/apn/-/apn-1.6.2.tgz", - "integrity": "sha1-wHTUEiC9t+ahlQIHXU/roZaDg/M=", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/apn/-/apn-2.2.0.tgz", + "integrity": "sha512-YIypYzPVJA9wzNBLKZ/mq2l1IZX/2FadPvwmSv4ZeR0VH7xdNITQ6Pucgh0Uw6ZZKC+XwheaJ57DFZAhJ0FvPg==", "requires": { - "q": "1.x" + "debug": "^3.1.0", + "http2": "https://github.com/node-apn/node-http2/archive/apn-2.1.4.tar.gz", + "jsonwebtoken": "^8.1.0", + "node-forge": "^0.7.1", + "verror": "^1.10.0" }, "dependencies": { - "q": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, @@ -17634,6 +17646,10 @@ "sshpk": "^1.7.0" } }, + "http2": { + "version": "https://github.com/node-apn/node-http2/archive/apn-2.1.4.tar.gz", + "integrity": "sha512-ad4u4I88X9AcUgxCRW3RLnbh7xHWQ1f5HbrXa7gEy2x4Xgq+rq+auGx5I+nUDE2YYuqteGIlbxrwQXkIaYTfnQ==" + }, "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", diff --git a/package.json b/package.json index bf4cc33e3ee02..4c4b49c33ef96 100644 --- a/package.json +++ b/package.json @@ -137,7 +137,7 @@ "@rocket.chat/ui-kit": "^0.7.1", "@slack/client": "^4.8.0", "adm-zip": "RocketChat/adm-zip", - "apn": "1.6.2", + "apn": "2.2.0", "archiver": "^3.0.0", "arraybuffer-to-string": "^1.0.2", "atlassian-crowd": "^0.5.0", From ca1a28e4f11f7dcc648111721ad482561fe943e3 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 17 Apr 2020 00:51:34 -0300 Subject: [PATCH 09/40] Unify part of the gateway and native send logic --- app/push/server/push.js | 243 +++++++++++++++------------------------- 1 file changed, 88 insertions(+), 155 deletions(-) diff --git a/app/push/server/push.js b/app/push/server/push.js index f926904a758d0..f5a76569be1ed 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -202,127 +202,28 @@ export class PushClass { appTokensCollection.rawCollection().updateMany({ token }, { $unset: { token: true } }); } - // Universal send function - _querySend(query, notification) { - const countApn = []; - const countGcm = []; - - appTokensCollection.find(query).forEach((app) => { - logger.debug('send to token', app.token); - - if (app.token.apn) { - countApn.push(app._id); - // Send to APN - if (this.options.apn) { - notification.topic = app.appName; - sendAPN(app.token.apn, notification); - } - } else if (app.token.gcm) { - countGcm.push(app._id); - - // Send to GCM - // We do support multiple here - so we should construct an array - // and send it bulk - Investigate limit count of id's - if (this.options.gcm && this.options.gcm.apiKey) { - sendGCM({ userTokens: app.token.gcm, notification, _replaceToken: this._replaceToken, _removeToken: this._removeToken, options: this.options }); - } - } else { - throw new Error('send got a faulty query'); + serverSendNative(app, notification, countApn, countGcm) { + logger.debug('send to token', app.token); + + if (app.token.apn) { + countApn.push(app._id); + // Send to APN + if (this.options.apn) { + notification.topic = app.appName; + sendAPN(app.token.apn, notification); } - }); - - if (LoggerManager.logLevel === 2) { - logger.debug(`Sent message "${ notification.title }" to ${ countApn.length } ios apps ${ countGcm.length } android apps`); - - // Add some verbosity about the send result, making sure the developer - // understands what just happened. - if (!countApn.length && !countGcm.length) { - if (appTokensCollection.find().count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" is empty - No clients have registered on the server yet...'); - } - } else if (!countApn.length) { - if (appTokensCollection.find({ 'token.apn': { $exists: true } }).count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" - No APN clients have registered on the server yet...'); - } - } else if (!countGcm.length) { - if (appTokensCollection.find({ 'token.gcm': { $exists: true } }).count() === 0) { - logger.debug('GUIDE: The "appTokensCollection" - No GCM clients have registered on the server yet...'); - } + } else if (app.token.gcm) { + countGcm.push(app._id); + + // Send to GCM + // We do support multiple here - so we should construct an array + // and send it bulk - Investigate limit count of id's + if (this.options.gcm && this.options.gcm.apiKey) { + sendGCM({ userTokens: app.token.gcm, notification, _replaceToken: this._replaceToken, _removeToken: this._removeToken, options: this.options }); } + } else { + throw new Error('send got a faulty query'); } - - return { - apn: countApn, - gcm: countGcm, - }; - } - - serverSendNative(notification) { - notification = notification || { badge: 0 }; - let query; - - // Check basic options - if (notification.from !== `${ notification.from }`) { - throw new Error('send: option "from" not a string'); - } - - if (notification.title !== `${ notification.title }`) { - throw new Error('send: option "title" not a string'); - } - - if (notification.text !== `${ notification.text }`) { - throw new Error('send: option "text" not a string'); - } - - if (notification.token || notification.tokens) { - // The user set one token or array of tokens - const tokenList = notification.token ? [notification.token] : notification.tokens; - - logger.debug(`Send message "${ notification.title }" via token(s)`, tokenList); - - query = { - $or: [ - // XXX: Test this query: can we hand in a list of push tokens? - { $and: [ - { token: { $in: tokenList } }, - // And is not disabled - { enabled: { $ne: false } }, - ], - }, - // XXX: Test this query: does this work on app id? - { $and: [ - { _id: { $in: tokenList } }, // one of the app ids - { $or: [ - { 'token.apn': { $exists: true } }, // got apn token - { 'token.gcm': { $exists: true } }, // got gcm token - ] }, - // And is not disabled - { enabled: { $ne: false } }, - ], - }, - ], - }; - } else if (notification.query) { - logger.debug(`Send message "${ notification.title }" via query`, notification.query); - - query = { - $and: [ - notification.query, // query object - { $or: [ - { 'token.apn': { $exists: true } }, // got apn token - { 'token.gcm': { $exists: true } }, // got gcm token - ] }, - // And is not disabled - { enabled: { $ne: false } }, - ], - }; - } - - if (query) { - // Convert to querySend and return status - return this._querySend(query, notification); - } - throw new Error('send: please set option "token"/"tokens" or "query"'); } sendGatewayPush(gateway, service, token, notification, tries = 0) { @@ -369,55 +270,87 @@ export class PushClass { }); } - serverSendGateway(notification = { badge: 0 }) { + serverSendGateway(app, notification, countApn, countGcm) { for (const gateway of this.options.gateways) { - if (notification.from !== String(notification.from)) { - throw new Error('Push.send: option "from" not a string'); - } - if (notification.title !== String(notification.title)) { - throw new Error('Push.send: option "title" not a string'); + logger.debug('send to token', app.token); + + if (app.token.apn) { + countApn.push(app._id); + notification.topic = app.appName; + return this.sendGatewayPush(gateway, 'apn', app.token.apn, notification); } - if (notification.text !== String(notification.text)) { - throw new Error('Push.send: option "text" not a string'); + + if (app.token.gcm) { + countGcm.push(app._id); + return this.sendGatewayPush(gateway, 'gcm', app.token.gcm, notification); } + } + } - logger.debug(`send message "${ notification.title }" via query`, notification.query); + serverSend(notification = { badge: 0 }) { + const countApn = []; + const countGcm = []; - const query = { - $and: [notification.query, { - $or: [{ - 'token.apn': { - $exists: true, - }, - }, { - 'token.gcm': { - $exists: true, - }, - }], + if (notification.from !== String(notification.from)) { + throw new Error('Push.send: option "from" not a string'); + } + if (notification.title !== String(notification.title)) { + throw new Error('Push.send: option "title" not a string'); + } + if (notification.text !== String(notification.text)) { + throw new Error('Push.send: option "text" not a string'); + } + + logger.debug(`send message "${ notification.title }" via query`, notification.query); + + const query = { + $and: [notification.query, { + $or: [{ + 'token.apn': { + $exists: true, + }, + }, { + 'token.gcm': { + $exists: true, + }, }], - }; + }], + }; - appTokensCollection.find(query).forEach((app) => { - logger.debug('send to token', app.token); + appTokensCollection.find(query).forEach((app) => { + logger.debug('send to token', app.token); - if (app.token.apn) { - notification.topic = app.appName; - return this.sendGatewayPush(gateway, 'apn', app.token.apn, notification); - } + if (this.options.gateways) { + return this.serverSendGateway(app, notification, countApn, countGcm); + } - if (app.token.gcm) { - return this.sendGatewayPush(gateway, 'gcm', app.token.gcm, notification); - } - }); - } - } + return this.serverSendNative(app, notification, countApn, countGcm); + }); + + if (LoggerManager.logLevel === 2) { + logger.debug(`Sent message "${ notification.title }" to ${ countApn.length } ios apps ${ countGcm.length } android apps`); - serverSend(notification) { - if (this.options.gateways) { - return this.serverSendGateway(notification); + // Add some verbosity about the send result, making sure the developer + // understands what just happened. + if (!countApn.length && !countGcm.length) { + if (appTokensCollection.find().count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" is empty - No clients have registered on the server yet...'); + } + } else if (!countApn.length) { + if (appTokensCollection.find({ 'token.apn': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No APN clients have registered on the server yet...'); + } + } else if (!countGcm.length) { + if (appTokensCollection.find({ 'token.gcm': { $exists: true } }).count() === 0) { + logger.debug('GUIDE: The "appTokensCollection" - No GCM clients have registered on the server yet...'); + } + } } - return this.serverSendNative(notification); + return { + apn: countApn, + gcm: countGcm, + }; } // This is a general function to validate that the data added to notifications From 14bb194b1ef5c26f8d0fa1e458646671904c732d Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 17 Apr 2020 01:16:47 -0300 Subject: [PATCH 10/40] Code improvements --- app/push/server/push.js | 215 +++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 122 deletions(-) diff --git a/app/push/server/push.js b/app/push/server/push.js index f5a76569be1ed..b4011cbe10b0e 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -11,13 +11,18 @@ import { logger, LoggerManager } from './logger'; export const _matchToken = Match.OneOf({ apn: String }, { gcm: String }); export const notificationsCollection = new Mongo.Collection('_raix_push_notifications'); export const appTokensCollection = new Mongo.Collection('_raix_push_app_tokens'); -appTokensCollection._ensureIndex({ userId: 1 }); -let isConfigured = false; +appTokensCollection._ensureIndex({ userId: 1 }); +notificationsCollection._ensureIndex({ createdAt: 1 }); +notificationsCollection._ensureIndex({ sent: 1 }); +notificationsCollection._ensureIndex({ sending: 1 }); +notificationsCollection._ensureIndex({ delayUntil: 1 }); export class PushClass { options = {} + isConfigured = false + configure(options) { this.options = Object.assign({ sendTimeout: 60000, // Timeout period for notification send @@ -36,17 +41,17 @@ export class PushClass { // $ openssl pkcs12 -in key.p12 -out key.pem -nodes // Block multiple calls - if (isConfigured) { + if (this.isConfigured) { throw new Error('Configure should not be called more than once!'); } - isConfigured = true; + this.isConfigured = true; logger.debug('Configure', this.options); if (this.options.apn) { initAPN({ options: this.options, _removeToken: this._removeToken }); - } // EO ios notification + } // This interval will allow only one notification to be sent at a time, it // will check for new notifications at every `options.sendInterval` @@ -69,123 +74,97 @@ export class PushClass { // let isSendingNotification = false; - if (this.options.sendInterval !== null) { - // This will require index since we sort notifications by createdAt - notificationsCollection._ensureIndex({ createdAt: 1 }); - notificationsCollection._ensureIndex({ sent: 1 }); - notificationsCollection._ensureIndex({ sending: 1 }); - notificationsCollection._ensureIndex({ delayUntil: 1 }); - - const sendNotification = (notification) => { - // Reserve notification - const now = +new Date(); - const timeoutAt = now + this.options.sendTimeout; - const reserved = notificationsCollection.update({ - _id: notification._id, - sent: false, // xxx: need to make sure this is set on create - sending: { $lt: now }, + const sendNotification = (notification) => { + // Reserve notification + const now = +new Date(); + const timeoutAt = now + this.options.sendTimeout; + const reserved = notificationsCollection.update({ + _id: notification._id, + sent: false, // xxx: need to make sure this is set on create + sending: { $lt: now }, + }, { + $set: { + sending: timeoutAt, }, - { - $set: { - sending: timeoutAt, - }, - }); - - // Make sure we only handle notifications reserved by this - // instance - if (reserved) { - // Check if query is set and is type String - if (notification.query && notification.query === `${ notification.query }`) { - try { - // The query is in string json format - we need to parse it - notification.query = JSON.parse(notification.query); - } catch (err) { - // Did the user tamper with this?? - throw new Error(`Error while parsing query string, Error: ${ err.message }`); - } + }); + + // Make sure we only handle notifications reserved by this instance + if (reserved) { + // Check if query is set and is type String + if (notification.query && notification.query === String(notification.query)) { + try { + // The query is in string json format - we need to parse it + notification.query = JSON.parse(notification.query); + } catch (err) { + // Did the user tamper with this?? + throw new Error(`Error while parsing query string, Error: ${ err.message }`); } - - // Send the notification - const result = this.serverSend(notification, this.options); - - if (!this.options.keepNotifications) { - // Pr. Default we will remove notifications - notificationsCollection.remove({ _id: notification._id }); - } else { - // Update the notification - notificationsCollection.update({ _id: notification._id }, { - $set: { - // Mark as sent - sent: true, - // Set the sent date - sentAt: new Date(), - // Count - count: result, - // Not being sent anymore - sending: 0, - }, - }); - } - - // Emit the send - // self.emit('send', { notification: notification._id, result }); - } // Else could not reserve - }; // EO sendNotification - - this.sendWorker(() => { - if (isSendingNotification) { - return; } - try { - // Set send fence - isSendingNotification = true; - - // var countSent = 0; - const batchSize = this.options.sendBatchSize || 1; - - const now = +new Date(); - - // Find notifications that are not being or already sent - const pendingNotifications = notificationsCollection.find({ $and: [ - // Message is not sent - { sent: false }, - // And not being sent by other instances - { sending: { $lt: now } }, - // And not queued for future - { $or: [ - { delayUntil: { $exists: false } }, - { delayUntil: { $lte: new Date() } }, - ], + // Send the notification + const result = this.serverSend(notification, this.options); + + if (!this.options.keepNotifications) { + // Pr. Default we will remove notifications + notificationsCollection.remove({ _id: notification._id }); + } else { + // Update the notification + notificationsCollection.update({ _id: notification._id }, { + $set: { + // Mark as sent + sent: true, + // Set the sent date + sentAt: new Date(), + // Count + count: result, + // Not being sent anymore + sending: 0, }, - ] }, { - // Sort by created date - sort: { createdAt: 1 }, - limit: batchSize, }); - - pendingNotifications.forEach((notification) => { - try { - sendNotification(notification); - } catch (error) { - logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); - } - }); // EO forEach - } finally { - // Remove the send fence - isSendingNotification = false; } - }, this.options.sendInterval || 15000); // Default every 15th sec - } else { - logger.debug('Send server is disabled'); - } + } + }; + + this.sendWorker(() => { + if (isSendingNotification) { + return; + } + + try { + // Set send fence + isSendingNotification = true; + + const batchSize = this.options.sendBatchSize || 1; + + // Find notifications that are not being or already sent + notificationsCollection.find({ + sent: false, + sending: { $lt: new Date() }, + $or: [ + { delayUntil: { $exists: false } }, + { delayUntil: { $lte: new Date() } }, + ], + }, { + sort: { createdAt: 1 }, + limit: batchSize, + }).forEach((notification) => { + try { + sendNotification(notification); + } catch (error) { + logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); + } + }); + } finally { + // Remove the send fence + isSendingNotification = false; + } + }, this.options.sendInterval || 15000); // Default every 15th sec } sendWorker(task, interval) { logger.debug(`Send worker started, using interval: ${ interval }`); - return Meteor.setInterval(function() { - // xxx: add exponential backoff on error + return Meteor.setInterval(() => { try { task(); } catch (error) { @@ -419,6 +398,8 @@ export class PushClass { const notification = Object.assign({ createdAt: new Date(), createdBy: currentUser, + sent: false, + sending: 0, }, _.pick(options, 'from', 'title', 'text')); // Add extra @@ -434,23 +415,13 @@ export class PushClass { // Set one token selector, this can be token, array of tokens or query if (options.query) { - // Set query to the json string version fixing #43 and #39 notification.query = JSON.stringify(options.query); - } else if (options.token) { - // Set token - notification.token = options.token; - } else if (options.tokens) { - // Set tokens - notification.tokens = options.tokens; } - // console.log(options); - if (typeof options.contentAvailable !== 'undefined') { + + if (options.contentAvailable != null) { notification.contentAvailable = options.contentAvailable; } - notification.sent = false; - notification.sending = 0; - // Validate the notification this._validateDocument(notification); From 006ecc703692c01d22930298044f097fe9327e4c Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 17 Apr 2020 10:35:50 -0300 Subject: [PATCH 11/40] Reduce meteor usage --- app/push/server/apn.js | 24 +++++++++++++----------- app/push/server/gcm.js | 29 +++++++++++++++++++++++------ app/push/server/methods.js | 5 +---- app/push/server/push.js | 16 +++++++++++++--- 4 files changed, 50 insertions(+), 24 deletions(-) diff --git a/app/push/server/apn.js b/app/push/server/apn.js index 351db7de28aca..6d254810f2dcb 100644 --- a/app/push/server/apn.js +++ b/app/push/server/apn.js @@ -1,6 +1,3 @@ -import { Meteor } from 'meteor/meteor'; -import { Match } from 'meteor/check'; -import { EJSON } from 'meteor/ejson'; import apn from 'apn'; import { logger } from './logger'; @@ -8,7 +5,7 @@ import { logger } from './logger'; let apnConnection; export const sendAPN = (userToken, notification) => { - if (Match.test(notification.apn, Object)) { + if (typeof notification.apn === 'object') { notification = Object.assign({}, notification, notification.apn); } @@ -40,10 +37,15 @@ export const sendAPN = (userToken, notification) => { if (notification.title != null) { note.alert.title = notification.title; + note.alert['summary-arg'] = notification.title; + } + + if (notification.notId != null) { + note['thread-id'] = notification.notId; } // Allow the user to set payload data - note.payload = notification.payload ? { ejson: EJSON.stringify(notification.payload) } : {}; + note.payload = notification.payload || {}; note.payload.messageFrom = notification.from; note.priority = priority; @@ -55,7 +57,7 @@ export const sendAPN = (userToken, notification) => { apnConnection.send(note, userToken); }; -export const initAPN = ({ options, _removeToken }) => { +export const initAPN = ({ options, _removeToken, absoluteUrl }) => { logger.debug('APN configured'); // Allow production to be a general option for push notifications @@ -82,15 +84,15 @@ export const initAPN = ({ options, _removeToken }) => { console.warn('WARNING: Push APN is in development mode'); } else if (options.apn.gateway === 'gateway.push.apple.com') { // In production - but warn if we are running on localhost - if (/http:\/\/localhost/.test(Meteor.absoluteUrl())) { + if (/http:\/\/localhost/.test(absoluteUrl)) { console.warn('WARNING: Push APN is configured to production mode - but server is running from localhost'); } } else { // Warn about gateways we dont know about - console.warn(`WARNING: Push APN unkown gateway "${ options.apn.gateway }"`); + console.warn(`WARNING: Push APN unknown gateway "${ options.apn.gateway }"`); } } else if (options.apn.production) { - if (/http:\/\/localhost/.test(Meteor.absoluteUrl())) { + if (/http:\/\/localhost/.test(absoluteUrl)) { console.warn('WARNING: Push APN is configured to production mode - but server is running from localhost'); } } else { @@ -111,7 +113,7 @@ export const initAPN = ({ options, _removeToken }) => { apnConnection = new apn.Provider(options.apn); // Listen to transmission errors - should handle the same way as feedback. - apnConnection.on('transmissionError', Meteor.bindEnvironment(function(errCode, notification/* , recipient*/) { + apnConnection.on('transmissionError', (errCode, notification/* , recipient*/) => { logger.debug('Got error code %d for token %s', errCode, notification.token); if ([2, 5, 8].indexOf(errCode) >= 0) { @@ -120,5 +122,5 @@ export const initAPN = ({ options, _removeToken }) => { apn: notification.token, }); } - })); + }); }; diff --git a/app/push/server/gcm.js b/app/push/server/gcm.js index 820f52cf67b65..cb37f3fc23283 100644 --- a/app/push/server/gcm.js +++ b/app/push/server/gcm.js @@ -1,17 +1,14 @@ -/* eslint-disable new-cap */ -import { Match } from 'meteor/check'; -import { EJSON } from 'meteor/ejson'; import gcm from 'node-gcm'; import { logger } from './logger'; export const sendGCM = function({ userTokens, notification, _replaceToken, _removeToken, options }) { - if (Match.test(notification.gcm, Object)) { + if (typeof notification.gcm === 'object') { notification = Object.assign({}, notification, notification.gcm); } // Make sure userTokens are an array of strings - if (Match.test(userTokens, String)) { + if (typeof userTokens === 'string') { userTokens = [userTokens]; } @@ -24,7 +21,7 @@ export const sendGCM = function({ userTokens, notification, _replaceToken, _remo logger.debug('sendGCM', userTokens, notification); // Allow user to set payload - const data = notification.payload ? { ejson: EJSON.stringify(notification.payload) } : {}; + const data = notification.payload || {}; data.title = notification.title; data.message = notification.text; @@ -34,6 +31,12 @@ export const sendGCM = function({ userTokens, notification, _replaceToken, _remo data.image = notification.image; } + if (notification.android_channel_id != null) { + data.android_channel_id = notification.android_channel_id; + } else { + logger.debug('For devices running Android 8.0 or later you are required to provide an android_channel_id. See https://github.com/raix/push/issues/341 for more info'); + } + // Set extra details if (notification.badge != null) { data.msgcnt = notification.badge; @@ -54,6 +57,20 @@ export const sendGCM = function({ userTokens, notification, _replaceToken, _remo data.picture = notification.picture; } + // Action Buttons + if (notification.actions != null) { + data.actions = notification.actions; + } + + // Force Start + if (notification.forceStart != null) { + data['force-start'] = notification.forceStart; + } + + if (notification.contentAvailable != null) { + data['content-available'] = notification.contentAvailable; + } + const message = new gcm.Message({ collapseKey: notification.from, // delayWhileIdle: true, diff --git a/app/push/server/methods.js b/app/push/server/methods.js index ac25aeacde2d5..cc6a26f74069b 100644 --- a/app/push/server/methods.js +++ b/app/push/server/methods.js @@ -73,10 +73,7 @@ Meteor.methods({ }); } - if (doc) { - // xxx: Hack - // Clean up mech making sure tokens are uniq - android sometimes generate - // new tokens resulting in duplicates + if (doc && doc.token) { const removed = appTokensCollection.remove({ $and: [ { _id: { $ne: doc._id } }, diff --git a/app/push/server/push.js b/app/push/server/push.js index b4011cbe10b0e..9f89a9a1531b1 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -1,4 +1,5 @@ import { Meteor } from 'meteor/meteor'; +import { EJSON } from 'meteor/ejson'; import { Match, check } from 'meteor/check'; import { Mongo } from 'meteor/mongo'; import { HTTP } from 'meteor/http'; @@ -50,7 +51,7 @@ export class PushClass { logger.debug('Configure', this.options); if (this.options.apn) { - initAPN({ options: this.options, _removeToken: this._removeToken }); + initAPN({ options: this.options, _removeToken: this._removeToken, absoluteUrl: Meteor.absoluteUrl() }); } // This interval will allow only one notification to be sent at a time, it @@ -184,6 +185,8 @@ export class PushClass { serverSendNative(app, notification, countApn, countGcm) { logger.debug('send to token', app.token); + notification.payload = notification.payload ? { ejson: EJSON.stringify(notification.payload) } : {}; + if (app.token.apn) { countApn.push(app._id); // Send to APN @@ -346,6 +349,7 @@ export class PushClass { sound: Match.Optional(String), notId: Match.Optional(Match.Integer), contentAvailable: Match.Optional(Match.Integer), + forceStart: Match.Optional(Match.Integer), apn: Match.Optional({ from: Match.Optional(String), title: Match.Optional(String), @@ -353,6 +357,7 @@ export class PushClass { badge: Match.Optional(Match.Integer), sound: Match.Optional(String), notId: Match.Optional(Match.Integer), + actions: Match.Optional([Match.Any]), category: Match.Optional(String), }), gcm: Match.Optional({ @@ -367,6 +372,7 @@ export class PushClass { sound: Match.Optional(String), notId: Match.Optional(Match.Integer), }), + android_channel_id: Match.Optional(String), query: Match.Optional(String), token: Match.Optional(_matchToken), tokens: Match.Optional([_matchToken]), @@ -403,14 +409,14 @@ export class PushClass { }, _.pick(options, 'from', 'title', 'text')); // Add extra - Object.assign(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil')); + Object.assign(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil', 'android_channel_id')); if (Match.test(options.apn, Object)) { notification.apn = _.pick(options.apn, 'from', 'title', 'text', 'badge', 'sound', 'notId', 'category'); } if (Match.test(options.gcm, Object)) { - notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId'); + notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId', 'actions', 'android_channel_id'); } // Set one token selector, this can be token, array of tokens or query @@ -422,6 +428,10 @@ export class PushClass { notification.contentAvailable = options.contentAvailable; } + if (options.forceStart != null) { + notification.forceStart = options.forceStart; + } + // Validate the notification this._validateDocument(notification); From ddb610f70aad144ddaac7ae0b1172822cf2fe74f Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 17 Apr 2020 10:38:02 -0300 Subject: [PATCH 12/40] Remove useless condition --- app/push/server/methods.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/app/push/server/methods.js b/app/push/server/methods.js index cc6a26f74069b..d6ed3e399bf02 100644 --- a/app/push/server/methods.js +++ b/app/push/server/methods.js @@ -73,7 +73,7 @@ Meteor.methods({ }); } - if (doc && doc.token) { + if (doc.token) { const removed = appTokensCollection.remove({ $and: [ { _id: { $ne: doc._id } }, @@ -88,13 +88,8 @@ Meteor.methods({ } } - if (doc) { - logger.debug('updated', doc); - } + logger.debug('updated', doc); - if (!doc) { - throw new Meteor.Error(500, 'setPushToken could not create record'); - } // Return the doc we want to use return doc; }, From a96a1b454956282d47b116f50eed3e6543e95d18 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 17 Apr 2020 11:49:26 -0300 Subject: [PATCH 13/40] Make APN work --- .meteor/versions | 2 -- app/push/server/apn.js | 34 ++++++++++++---------------------- app/push/server/push.js | 7 +++++-- server/lib/pushConfig.js | 10 +++++----- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/.meteor/versions b/.meteor/versions index edfb768fcbb44..02e97dbc384a3 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -114,7 +114,6 @@ pauli:accounts-linkedin@5.0.0 pauli:linkedin-oauth@5.0.0 promise@0.11.2 raix:eventemitter@0.1.3 -raix:eventstate@0.0.4 raix:handlebar-helpers@0.2.5 raix:ui-dropped-event@0.0.7 random@1.1.0 @@ -127,7 +126,6 @@ rocketchat:i18n@0.0.1 rocketchat:livechat@0.0.1 rocketchat:mongo-config@0.0.1 rocketchat:oauth2-server@2.1.0 -rocketchat:push@3.3.1 rocketchat:streamer@1.1.0 rocketchat:tap-i18n@1.9.1 rocketchat:version@1.0.0 diff --git a/app/push/server/apn.js b/app/push/server/apn.js index 6d254810f2dcb..54171e106f1bf 100644 --- a/app/push/server/apn.js +++ b/app/push/server/apn.js @@ -14,12 +14,9 @@ export const sendAPN = (userToken, notification) => { const note = new apn.Notification(); note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now. - if (notification.badge != null) { - note.badge = notification.badge; - } - if (notification.sound != null) { - note.sound = notification.sound; - } + note.badge = notification.badge; + note.sound = notification.sound; + if (notification.contentAvailable != null) { note.setContentAvailable(notification.contentAvailable); } @@ -27,21 +24,13 @@ export const sendAPN = (userToken, notification) => { // adds category support for iOS8 custom actions as described here: // https://developer.apple.com/library/ios/documentation/NetworkingInternet/Conceptual/ // RemoteNotificationsPG/Chapters/IPhoneOSClientImp.html#//apple_ref/doc/uid/TP40008194-CH103-SW36 - if (notification.category != null) { - note.category = notification.category; - } - - note.alert = { - body: notification.text, - }; + note.category = notification.category; - if (notification.title != null) { - note.alert.title = notification.title; - note.alert['summary-arg'] = notification.title; - } + note.body = notification.text; + note.title = notification.title; if (notification.notId != null) { - note['thread-id'] = notification.notId; + note.threadId = String(notification.notId); } // Allow the user to set payload data @@ -53,6 +42,7 @@ export const sendAPN = (userToken, notification) => { // Store the token on the note so we can reference it if there was an error note.token = userToken; note.topic = notification.topic; + note.mutableContent = 1; apnConnection.send(note, userToken); }; @@ -100,13 +90,13 @@ export const initAPN = ({ options, _removeToken, absoluteUrl }) => { } // Check certificate data - if (!options.apn.certData || !options.apn.certData.length) { - console.error('ERROR: Push server could not find certData'); + if (!options.apn.cert || !options.apn.cert.length) { + console.error('ERROR: Push server could not find cert'); } // Check key data - if (!options.apn.keyData || !options.apn.keyData.length) { - console.error('ERROR: Push server could not find keyData'); + if (!options.apn.key || !options.apn.key.length) { + console.error('ERROR: Push server could not find key'); } // Rig apn connection diff --git a/app/push/server/push.js b/app/push/server/push.js index 9f89a9a1531b1..13948830d7adc 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -76,8 +76,10 @@ export class PushClass { let isSendingNotification = false; const sendNotification = (notification) => { + logger.debug('Sending notification', notification); + // Reserve notification - const now = +new Date(); + const now = Date.now(); const timeoutAt = now + this.options.sendTimeout; const reserved = notificationsCollection.update({ _id: notification._id, @@ -140,7 +142,7 @@ export class PushClass { // Find notifications that are not being or already sent notificationsCollection.find({ sent: false, - sending: { $lt: new Date() }, + sending: { $lt: Date.now() }, $or: [ { delayUntil: { $exists: false } }, { delayUntil: { $lte: new Date() } }, @@ -153,6 +155,7 @@ export class PushClass { sendNotification(notification); } catch (error) { logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); + logger.debug(error.stack); } }); } finally { diff --git a/server/lib/pushConfig.js b/server/lib/pushConfig.js index b85bbea481ba3..e1828bf2cfe80 100644 --- a/server/lib/pushConfig.js +++ b/server/lib/pushConfig.js @@ -86,20 +86,20 @@ function configurePush() { apn = { passphrase: settings.get('Push_apn_passphrase'), - keyData: settings.get('Push_apn_key'), - certData: settings.get('Push_apn_cert'), + key: settings.get('Push_apn_key'), + cert: settings.get('Push_apn_cert'), }; if (settings.get('Push_production') !== true) { apn = { passphrase: settings.get('Push_apn_dev_passphrase'), - keyData: settings.get('Push_apn_dev_key'), - certData: settings.get('Push_apn_dev_cert'), + key: settings.get('Push_apn_dev_key'), + cert: settings.get('Push_apn_dev_cert'), gateway: 'gateway.sandbox.push.apple.com', }; } - if (!apn.keyData || apn.keyData.trim() === '' || !apn.certData || apn.certData.trim() === '') { + if (!apn.key || apn.key.trim() === '' || !apn.cert || apn.cert.trim() === '') { apn = undefined; } From 42bbd432e038bc91de951e00a6f756c24803fd07 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 17 Apr 2020 13:02:18 -0300 Subject: [PATCH 14/40] Fix misspeling --- app/push/server/gcm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/push/server/gcm.js b/app/push/server/gcm.js index cb37f3fc23283..5d7c11d04cb77 100644 --- a/app/push/server/gcm.js +++ b/app/push/server/gcm.js @@ -97,7 +97,7 @@ export const sendGCM = function({ userTokens, notification, _replaceToken, _remo return; } - logger.debuglog(`ANDROID: Result of sender: ${ JSON.stringify(result) }`); + logger.debug(`ANDROID: Result of sender: ${ JSON.stringify(result) }`); if (result.canonical_ids === 1 && userToken) { // This is an old device, token is replaced From 9de7e679f08de602786f6491b399d68ff6503805 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 00:45:39 -0300 Subject: [PATCH 15/40] Start queue --- .../server/functions/notifications/email.js | 23 ++- .../server/functions/notifications/mobile.js | 26 +-- .../server/lib/sendNotificationsOnMessage.js | 42 +++-- app/models/server/models/NotificationQueue.js | 11 ++ app/models/server/raw/BaseRaw.js | 9 +- app/models/server/raw/NotificationQueue.js | 5 + app/models/server/raw/index.js | 3 + .../server/NotificationQueue.ts | 166 ++++++++++++++++++ package.json | 2 + 9 files changed, 256 insertions(+), 31 deletions(-) create mode 100644 app/models/server/models/NotificationQueue.js create mode 100644 app/models/server/raw/NotificationQueue.js create mode 100644 app/notification-queue/server/NotificationQueue.ts diff --git a/app/lib/server/functions/notifications/email.js b/app/lib/server/functions/notifications/email.js index 13a4652072d38..2c42c65f0986b 100644 --- a/app/lib/server/functions/notifications/email.js +++ b/app/lib/server/functions/notifications/email.js @@ -110,7 +110,7 @@ const getButtonUrl = (room, subscription, message) => { }); }; -export function sendEmail({ message, user, subscription, room, emailAddress, hasMentionToUser }) { +export function getEmailData({ message, user, subscription, room, emailAddress, hasMentionToUser }) { const username = settings.get('UI_Use_Real_Name') ? message.u.name || message.u.username : message.u.username; let subjectKey = 'Offline_Mention_All_Email'; @@ -152,12 +152,21 @@ export function sendEmail({ message, user, subscription, room, emailAddress, has } metrics.notificationsSent.inc({ notification_type: 'email' }); - return Mailer.send(email); + return email; +} + +export function sendEmailFromData(data) { + metrics.notificationsSent.inc({ notification_type: 'email' }); + return Mailer.send(data); +} + +export function sendEmail({ message, user, subscription, room, emailAddress, hasMentionToUser }) { + return sendEmailFromData(getEmailData({ message, user, subscription, room, emailAddress, hasMentionToUser })); } export function shouldNotifyEmail({ disableAllMessageNotifications, - statusConnection, + // statusConnection, emailNotifications, isHighlighted, hasMentionToUser, @@ -170,10 +179,10 @@ export function shouldNotifyEmail({ return false; } - // use connected (don't need to send him an email) - if (statusConnection === 'online') { - return false; - } + // // use connected (don't need to send him an email) + // if (statusConnection === 'online') { + // return false; + // } // user/room preference to nothing if (emailNotifications === 'nothing') { diff --git a/app/lib/server/functions/notifications/mobile.js b/app/lib/server/functions/notifications/mobile.js index 1e7a5d54849d3..15f793c25c7e9 100644 --- a/app/lib/server/functions/notifications/mobile.js +++ b/app/lib/server/functions/notifications/mobile.js @@ -8,10 +8,10 @@ import { PushNotification } from '../../../../push-notifications/server'; const CATEGORY_MESSAGE = 'MESSAGE'; const CATEGORY_MESSAGE_NOREPLY = 'MESSAGE_NOREPLY'; -let alwaysNotifyMobileBoolean; -settings.get('Notifications_Always_Notify_Mobile', (key, value) => { - alwaysNotifyMobileBoolean = value; -}); +// let alwaysNotifyMobileBoolean; +// settings.get('Notifications_Always_Notify_Mobile', (key, value) => { +// alwaysNotifyMobileBoolean = value; +// }); let SubscriptionRaw; Meteor.startup(() => { @@ -46,13 +46,13 @@ function enableNotificationReplyButton(room, username) { return !room.muted.includes(username); } -export async function sendSinglePush({ room, message, userId, receiverUsername, senderUsername, senderName, notificationMessage }) { +export async function getPushData({ room, message, userId, receiverUsername, senderUsername, senderName, notificationMessage }) { let username = ''; if (settings.get('Push_show_username_room')) { username = settings.get('UI_Use_Real_Name') === true ? senderName : senderUsername; } - PushNotification.send({ + return { roomId: message.rid, payload: { host: Meteor.absoluteUrl(), @@ -71,7 +71,11 @@ export async function sendSinglePush({ room, message, userId, receiverUsername, userId, }, category: enableNotificationReplyButton(room, receiverUsername) ? CATEGORY_MESSAGE : CATEGORY_MESSAGE_NOREPLY, - }); + }; +} + +export async function sendSinglePush({ room, message, userId, receiverUsername, senderUsername, senderName, notificationMessage }) { + PushNotification.send(getPushData({ room, message, userId, receiverUsername, senderUsername, senderName, notificationMessage })); } export function shouldNotifyMobile({ @@ -81,7 +85,7 @@ export function shouldNotifyMobile({ isHighlighted, hasMentionToUser, hasReplyToThread, - statusConnection, + // statusConnection, roomType, }) { if (disableAllMessageNotifications && mobilePushNotifications == null && !isHighlighted && !hasMentionToUser && !hasReplyToThread) { @@ -92,9 +96,9 @@ export function shouldNotifyMobile({ return false; } - if (!alwaysNotifyMobileBoolean && statusConnection === 'online') { - return false; - } + // if (!alwaysNotifyMobileBoolean && statusConnection === 'online') { + // return false; + // } if (!mobilePushNotifications) { if (settings.get('Accounts_Default_User_Preferences_mobileNotifications') === 'all') { diff --git a/app/lib/server/lib/sendNotificationsOnMessage.js b/app/lib/server/lib/sendNotificationsOnMessage.js index 1d2e831c44418..ffd68e8bc8c68 100644 --- a/app/lib/server/lib/sendNotificationsOnMessage.js +++ b/app/lib/server/lib/sendNotificationsOnMessage.js @@ -7,10 +7,11 @@ import { callbacks } from '../../../callbacks/server'; import { Subscriptions, Users } from '../../../models/server'; import { roomTypes } from '../../../utils'; import { callJoinRoom, messageContainsHighlight, parseMessageTextPerUser, replaceMentionedUsernamesWithFullNames } from '../functions/notifications'; -import { sendEmail, shouldNotifyEmail } from '../functions/notifications/email'; -import { sendSinglePush, shouldNotifyMobile } from '../functions/notifications/mobile'; +import { getEmailData, shouldNotifyEmail } from '../functions/notifications/email'; +import { getPushData, shouldNotifyMobile } from '../functions/notifications/mobile'; import { notifyDesktopUser, shouldNotifyDesktop } from '../functions/notifications/desktop'; import { notifyAudioUser, shouldNotifyAudio } from '../functions/notifications/audio'; +import { Notification } from '../../../notification-queue/server/NotificationQueue'; let TroubleshootDisableNotifications; @@ -115,6 +116,8 @@ export const sendNotification = async ({ }); } + const queueItems = []; + if (shouldNotifyMobile({ disableAllMessageNotifications, mobilePushNotifications, @@ -125,14 +128,17 @@ export const sendNotification = async ({ statusConnection: receiver.statusConnection, roomType, })) { - sendSinglePush({ - notificationMessage, - room, - message, - userId: subscription.u._id, - senderUsername: sender.username, - senderName: sender.name, - receiverUsername: receiver.username, + queueItems.push({ + type: 'push', + data: await getPushData({ + notificationMessage, + room, + message, + userId: subscription.u._id, + senderUsername: sender.username, + senderName: sender.name, + receiverUsername: receiver.username, + }), }); } @@ -148,13 +154,26 @@ export const sendNotification = async ({ })) { receiver.emails.some((email) => { if (email.verified) { - sendEmail({ message, receiver, subscription, room, emailAddress: email.address, hasMentionToUser }); + queueItems.push({ + type: 'email', + data: getEmailData({ message, receiver, subscription, room, emailAddress: email.address, hasMentionToUser }), + }); return true; } return false; }); } + + if (queueItems.length) { + Notification.collection.insert({ + uid: subscription.u._id, + rid: room._id, + sid: subscription._id, + ts: new Date(), + items: queueItems, + }); + } }; const project = { @@ -306,6 +325,7 @@ export async function sendAllNotifications(message, room) { return message; } // skips this callback if the message was edited + // TODO: Since we are queueing we could change the message on the queue if (message.editedAt) { return message; } diff --git a/app/models/server/models/NotificationQueue.js b/app/models/server/models/NotificationQueue.js new file mode 100644 index 0000000000000..b54eeadc9a85b --- /dev/null +++ b/app/models/server/models/NotificationQueue.js @@ -0,0 +1,11 @@ +import { Base } from './_Base'; + +export class NotificationQueue extends Base { + constructor() { + super('notification_queue'); + this.tryEnsureIndex({ sid: 1 }); + // this.tryEnsureIndex({ 'room._id': 1, date: 1 }, { unique: true }); + } +} + +export default new NotificationQueue(); diff --git a/app/models/server/raw/BaseRaw.js b/app/models/server/raw/BaseRaw.js index 1a65a2c9cebce..15a80bb37bb6a 100644 --- a/app/models/server/raw/BaseRaw.js +++ b/app/models/server/raw/BaseRaw.js @@ -1,3 +1,5 @@ +import { ObjectId } from 'mongodb'; + export class BaseRaw { constructor(col) { this.col = col; @@ -19,8 +21,11 @@ export class BaseRaw { return this.col.find(...args); } - insert(...args) { - return this.col.insert(...args); + insert(docs, options) { + if (!Array.isArray(docs) && !docs._id) { + docs._id = new ObjectId().toString(); + } + return this.col.insert(docs, options); } update(...args) { diff --git a/app/models/server/raw/NotificationQueue.js b/app/models/server/raw/NotificationQueue.js new file mode 100644 index 0000000000000..56403d2833822 --- /dev/null +++ b/app/models/server/raw/NotificationQueue.js @@ -0,0 +1,5 @@ +import { BaseRaw } from './BaseRaw'; + +export class NotificationQueueRaw extends BaseRaw { + +} diff --git a/app/models/server/raw/index.js b/app/models/server/raw/index.js index 042900293a67c..ed74d56e7ec05 100644 --- a/app/models/server/raw/index.js +++ b/app/models/server/raw/index.js @@ -46,6 +46,8 @@ import LivechatAgentActivityModel from '../models/LivechatAgentActivity'; import { LivechatAgentActivityRaw } from './LivechatAgentActivity'; import StatisticsModel from '../models/Statistics'; import { StatisticsRaw } from './Statistics'; +import NotificationQueueModel from '../models/NotificationQueue'; +import { NotificationQueueRaw } from './NotificationQueue'; export const Permissions = new PermissionsRaw(PermissionsModel.model.rawCollection()); export const Roles = new RolesRaw(RolesModel.model.rawCollection()); @@ -71,3 +73,4 @@ export const CustomSounds = new CustomSoundsRaw(CustomSoundsModel.model.rawColle export const CustomUserStatus = new CustomUserStatusRaw(CustomUserStatusModel.model.rawCollection()); export const LivechatAgentActivity = new LivechatAgentActivityRaw(LivechatAgentActivityModel.model.rawCollection()); export const Statistics = new StatisticsRaw(StatisticsModel.model.rawCollection()); +export const NotificationQueue = new NotificationQueueRaw(NotificationQueueModel.model.rawCollection()); diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts new file mode 100644 index 0000000000000..274f87e58116b --- /dev/null +++ b/app/notification-queue/server/NotificationQueue.ts @@ -0,0 +1,166 @@ +import { Collection } from 'mongodb'; + +import { NotificationQueue } from '../../models/server/raw'; +import { sendEmailFromData } from '../../lib/server/functions/notifications/email'; +import { PushNotification } from '../../push-notifications/server'; + +interface INotificationItemPush { + type: 'push'; + data: { + roomId: string; + payload: { + host: string; + rid: string; + sender: { + _id: string; + username: string; + name?: string; + }; + type: string; + messageId: string; + }; + roomName: string; + username: string; + message: string; + badge: number; + usersTo: { + userId: string; + }; + category: string; + }; +} + +interface INotificationItemEmail { + type: 'email'; + data: { + to: string; + subject: string; + html: string; + data: { + room_path: string; + }; + from: string; + }; +} + +type NotificationItem = INotificationItemPush | INotificationItemEmail; + +interface INotification { + _id: string; + uid: string; + rid: string; + sid: string; + ts: Date; + sending: Date; + items: NotificationItem[]; +} + +class NotificationClass { + public readonly collection: Collection = NotificationQueue.col + + private running = false; + + private delay = 2000; + + private maxBatchSize = 5; + + private resetPendingInterval?: NodeJS.Timer; + + private resetPendingDelay = 5 * 60 * 1000; + + initWorker(): void { + this.running = true; + this.executeWorkerLater(); + + this.resetPendingInterval = setInterval(() => { + const date = new Date(); + date.setMinutes(date.getMinutes() - 5); + this.collection.updateMany({ + sending: { $lt: date }, + }, { + $unset: { + sending: 1, + }, + }); + }, this.resetPendingDelay); + } + + stopWorker(): void { + this.running = false; + if (this.resetPendingInterval) { + clearInterval(this.resetPendingInterval); + } + } + + executeWorkerLater(): void { + if (!this.running) { + return; + } + + setTimeout(this.worker.bind(this), this.delay); + } + + async worker(counter = 0): Promise { + console.log('working'); + const notification = (await this.collection.findOneAndUpdate({ + sending: { $exists: false }, + ts: { $lt: new Date() }, + }, { + $set: { + sending: new Date(), + }, + }, { + sort: { + ts: -1, + }, + })).value; + + if (!notification) { + return this.executeWorkerLater(); + } + + console.log('processing', notification._id); + + try { + for (const item of notification.items) { + switch (item.type) { + case 'push': + this.push(item); + break; + // case 'email': + // this.email(item); + // break; + } + } + + this.collection.deleteOne({ + _id: notification._id, + }); + } catch (e) { + console.error(e); + this.collection.updateOne({ + _id: notification._id, + }, { + $unset: { + sending: 1, + }, + }); + } + + if (counter >= this.maxBatchSize) { + return this.executeWorkerLater(); + } + this.worker(counter++); + } + + push(item: INotificationItemPush): void { + PushNotification.send(item.data); + } + + email(item: INotificationItemEmail): void { + sendEmailFromData(item.data); + } +} + +export const Notification = new NotificationClass(); +Notification.initWorker(); diff --git a/package.json b/package.json index d2d39be42f58d..0b29dab85bdab 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "@storybook/react": "^5.2.8", "@types/bcrypt": "^3.0.0", "@types/meteor": "^1.4.37", + "@types/mongodb": "^3.5.8", "@typescript-eslint/eslint-plugin": "^2.11.0", "@typescript-eslint/parser": "^2.11.0", "acorn": "^6.4.1", @@ -202,6 +203,7 @@ "mkdirp": "^0.5.1", "moment": "^2.22.2", "moment-timezone": "^0.5.27", + "mongodb": "^3.5.6", "node-dogstatsd": "^0.0.7", "node-gcm": "0.14.4", "node-rsa": "^1.0.5", From afeff3d30a7cdb6a92d304d694082ce98dafba22 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 01:25:54 -0300 Subject: [PATCH 16/40] First implementation of notification schedule --- .../server/lib/sendNotificationsOnMessage.js | 3 +- .../server/NotificationQueue.ts | 47 +++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/app/lib/server/lib/sendNotificationsOnMessage.js b/app/lib/server/lib/sendNotificationsOnMessage.js index ffd68e8bc8c68..9013c4eec0c6c 100644 --- a/app/lib/server/lib/sendNotificationsOnMessage.js +++ b/app/lib/server/lib/sendNotificationsOnMessage.js @@ -166,11 +166,10 @@ export const sendNotification = async ({ } if (queueItems.length) { - Notification.collection.insert({ + Notification.scheduleItem({ uid: subscription.u._id, rid: room._id, sid: subscription._id, - ts: new Date(), items: queueItems, }); } diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index 274f87e58116b..7ae703ac72862 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -1,9 +1,11 @@ -import { Collection } from 'mongodb'; +import { Collection, ObjectId } from 'mongodb'; -import { NotificationQueue } from '../../models/server/raw'; +import { NotificationQueue, Subscriptions } from '../../models/server/raw'; import { sendEmailFromData } from '../../lib/server/functions/notifications/email'; import { PushNotification } from '../../push-notifications/server'; +const SubscriptionsCollection: Collection = Subscriptions.col; + interface INotificationItemPush { type: 'push'; data: { @@ -51,7 +53,7 @@ interface INotification { rid: string; sid: string; ts: Date; - sending: Date; + sending?: Date; items: NotificationItem[]; } @@ -160,6 +162,45 @@ class NotificationClass { email(item: INotificationItemEmail): void { sendEmailFromData(item.data); } + + async scheduleItem({ uid, rid, sid, items }: {uid: string; rid: string; sid: string; items: NotificationItem[]}): Promise { + // TODO: ls only changes when the channel is marked as read or when using ESC + // need to find another way + const sub = await SubscriptionsCollection.findOne({ + 'u._id': uid, + }, { + projection: { + ls: 1, + }, + sort: { + ls: -1, + }, + }); + + const delay = 120; + + const ts = new Date(); + + if (sub?.ls) { + const elapsedSeconds = Math.floor((Date.now() - sub.ls) / 1000); + if (elapsedSeconds < delay) { + console.log(delay - elapsedSeconds); + ts.setSeconds(ts.getSeconds() + delay - elapsedSeconds); + } + } else { + console.log(delay); + ts.setSeconds(ts.getSeconds() + delay); + } + + await this.collection.insertOne({ + _id: new ObjectId().toString(), + uid, + rid, + sid, + ts, + items, + }); + } } export const Notification = new NotificationClass(); From a653672d95e0fb071244f6d35cd1588fa0383df9 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 15:35:48 -0300 Subject: [PATCH 17/40] Improve logic --- .../server/NotificationQueue.ts | 59 ++++++++++++------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index 7ae703ac72862..c9d877dbcda8e 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -1,10 +1,10 @@ import { Collection, ObjectId } from 'mongodb'; -import { NotificationQueue, Subscriptions } from '../../models/server/raw'; +import { NotificationQueue, Users } from '../../models/server/raw'; import { sendEmailFromData } from '../../lib/server/functions/notifications/email'; import { PushNotification } from '../../push-notifications/server'; -const SubscriptionsCollection: Collection = Subscriptions.col; +const UsersCollection: Collection = Users.col; interface INotificationItemPush { type: 'push'; @@ -53,6 +53,7 @@ interface INotification { rid: string; sid: string; ts: Date; + schedule?: Date; sending?: Date; items: NotificationItem[]; } @@ -103,17 +104,19 @@ class NotificationClass { } async worker(counter = 0): Promise { - console.log('working'); const notification = (await this.collection.findOneAndUpdate({ sending: { $exists: false }, - ts: { $lt: new Date() }, + $or: [ + { schedule: { $exists: false } }, + { schedule: { $lte: new Date() } }, + ], }, { $set: { sending: new Date(), }, }, { sort: { - ts: -1, + ts: 1, }, })).value; @@ -121,6 +124,16 @@ class NotificationClass { return this.executeWorkerLater(); } + // Once we start notifying the user we anticipate all the schedules + this.collection.updateMany({ + uid: notification.uid, + schedule: { $exists: true }, + }, { + $unset: { + schedule: 1, + }, + }); + console.log('processing', notification._id); try { @@ -164,32 +177,33 @@ class NotificationClass { } async scheduleItem({ uid, rid, sid, items }: {uid: string; rid: string; sid: string; items: NotificationItem[]}): Promise { - // TODO: ls only changes when the channel is marked as read or when using ESC - // need to find another way - const sub = await SubscriptionsCollection.findOne({ - 'u._id': uid, + const user = await UsersCollection.findOne({ + _id: uid, }, { projection: { - ls: 1, - }, - sort: { - ls: -1, + statusConnection: 1, + _updatedAt: 1, }, }); + if (!user) { + return; + } + + // TODO: Make it configurable const delay = 120; - const ts = new Date(); + let schedule: Date | undefined; - if (sub?.ls) { - const elapsedSeconds = Math.floor((Date.now() - sub.ls) / 1000); + if (user.statusConnection === 'online') { + schedule = new Date(); + schedule.setSeconds(schedule.getSeconds() + delay); + } else if (user.statusConnection === 'away') { + const elapsedSeconds = Math.floor((Date.now() - user._updatedAt) / 1000); if (elapsedSeconds < delay) { - console.log(delay - elapsedSeconds); - ts.setSeconds(ts.getSeconds() + delay - elapsedSeconds); + schedule = new Date(); + schedule.setSeconds(schedule.getSeconds() + delay - elapsedSeconds); } - } else { - console.log(delay); - ts.setSeconds(ts.getSeconds() + delay); } await this.collection.insertOne({ @@ -197,7 +211,8 @@ class NotificationClass { uid, rid, sid, - ts, + ts: new Date(), + schedule, items, }); } From faa43f0950d62d482a2ce12b6ddef9f2f99de204 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 16:02:17 -0300 Subject: [PATCH 18/40] Reset queue on message read --- .../server/lib/sendNotificationsOnMessage.js | 1 - .../server/NotificationQueue.ts | 68 +++++--- package-lock.json | 160 ++++++++++++++++-- server/methods/readMessages.js | 3 + 4 files changed, 187 insertions(+), 45 deletions(-) diff --git a/app/lib/server/lib/sendNotificationsOnMessage.js b/app/lib/server/lib/sendNotificationsOnMessage.js index 9013c4eec0c6c..05385fc67c1e9 100644 --- a/app/lib/server/lib/sendNotificationsOnMessage.js +++ b/app/lib/server/lib/sendNotificationsOnMessage.js @@ -324,7 +324,6 @@ export async function sendAllNotifications(message, room) { return message; } // skips this callback if the message was edited - // TODO: Since we are queueing we could change the message on the queue if (message.editedAt) { return message; } diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index c9d877dbcda8e..e384956539189 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -104,35 +104,14 @@ class NotificationClass { } async worker(counter = 0): Promise { - const notification = (await this.collection.findOneAndUpdate({ - sending: { $exists: false }, - $or: [ - { schedule: { $exists: false } }, - { schedule: { $lte: new Date() } }, - ], - }, { - $set: { - sending: new Date(), - }, - }, { - sort: { - ts: 1, - }, - })).value; + const notification = await this.getNextNotification(); if (!notification) { return this.executeWorkerLater(); } // Once we start notifying the user we anticipate all the schedules - this.collection.updateMany({ - uid: notification.uid, - schedule: { $exists: true }, - }, { - $unset: { - schedule: 1, - }, - }); + this.flushQueueForUser(notification.uid); console.log('processing', notification._id); @@ -142,9 +121,9 @@ class NotificationClass { case 'push': this.push(item); break; - // case 'email': - // this.email(item); - // break; + case 'email': + this.email(item); + break; } } @@ -168,6 +147,35 @@ class NotificationClass { this.worker(counter++); } + async flushQueueForUser(userId: string): Promise { + await this.collection.updateMany({ + uid: userId, + schedule: { $exists: true }, + }, { + $unset: { + schedule: 1, + }, + }); + } + + async getNextNotification(): Promise { + return (await this.collection.findOneAndUpdate({ + sending: { $exists: false }, + $or: [ + { schedule: { $exists: false } }, + { schedule: { $lte: new Date() } }, + ], + }, { + $set: { + sending: new Date(), + }, + }, { + sort: { + ts: 1, + }, + })).value; + } + push(item: INotificationItemPush): void { PushNotification.send(item.data); } @@ -216,6 +224,14 @@ class NotificationClass { items, }); } + + async clearQueueForUser(userId: string): Promise { + const op = await this.collection.deleteMany({ + uid: userId, + }); + + return op.deletedCount; + } } export const Notification = new NotificationClass(); diff --git a/package-lock.json b/package-lock.json index a8b9a372490bb..24ac4d70013ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,50 @@ "requires": { "lodash": "^4.17.4", "mongodb": "^2.2.22" + }, + "dependencies": { + "es6-promise": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.2.1.tgz", + "integrity": "sha1-7FYjOGgDKQkgcXDDlEjiREndH8Q=" + }, + "mongodb": { + "version": "2.2.36", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.2.36.tgz", + "integrity": "sha512-P2SBLQ8Z0PVx71ngoXwo12+FiSfbNfGOClAao03/bant5DgLNkOPAck5IaJcEk4gKlQhDEURzfR3xuBG1/B+IA==", + "requires": { + "es6-promise": "3.2.1", + "mongodb-core": "2.1.20", + "readable-stream": "2.2.7" + } + }, + "process-nextick-args": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", + "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + }, + "readable-stream": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.7.tgz", + "integrity": "sha1-BwV6y+JGeyIELTb5jFrVBwVOlbE=", + "requires": { + "buffer-shims": "~1.0.0", + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "string_decoder": "~1.0.0", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", + "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "requires": { + "safe-buffer": "~5.1.0" + } + } } }, "@accounts/server": { @@ -6142,6 +6186,15 @@ "@types/node": "*" } }, + "@types/bson": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/bson/-/bson-4.0.2.tgz", + "integrity": "sha512-+uWmsejEHfmSjyyM/LkrP0orfE2m5Mx9Xel4tXNeqi1ldK5XMQcDsFkBmLDtuyKUbxj2jGDo0H240fbCRJZo7Q==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/caseless": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz", @@ -6275,6 +6328,16 @@ "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", "integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==" }, + "@types/mongodb": { + "version": "3.5.8", + "resolved": "https://registry.npmjs.org/@types/mongodb/-/mongodb-3.5.8.tgz", + "integrity": "sha512-2yOociZaXyiJ9CvGp/svjtlMCIPdl82XIRVmx35ehuWA046bipLwwcXfwVyvTYIU98yWYK5p44knCVQ+ZS4Bdw==", + "dev": true, + "requires": { + "@types/bson": "*", + "@types/node": "*" + } + }, "@types/node": { "version": "9.6.40", "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.40.tgz", @@ -12906,6 +12969,11 @@ "integrity": "sha1-OjYof1A05pnnV3kBBSwubJQlFjE=", "dev": true }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -20694,6 +20762,12 @@ "readable-stream": "^2.0.1" } }, + "memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "optional": true + }, "mensch": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/mensch/-/mensch-0.3.3.tgz", @@ -22105,38 +22179,40 @@ "ms": "^2.1.1" } }, + "es6-promise": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.2.1.tgz", + "integrity": "sha1-7FYjOGgDKQkgcXDDlEjiREndH8Q=", + "dev": true + }, + "mongodb": { + "version": "2.2.36", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.2.36.tgz", + "integrity": "sha512-P2SBLQ8Z0PVx71ngoXwo12+FiSfbNfGOClAao03/bant5DgLNkOPAck5IaJcEk4gKlQhDEURzfR3xuBG1/B+IA==", + "dev": true, + "requires": { + "es6-promise": "3.2.1", + "mongodb-core": "2.1.20", + "readable-stream": "2.2.7" + } + }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", "dev": true - } - } - }, - "mongodb": { - "version": "2.2.36", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.2.36.tgz", - "integrity": "sha512-P2SBLQ8Z0PVx71ngoXwo12+FiSfbNfGOClAao03/bant5DgLNkOPAck5IaJcEk4gKlQhDEURzfR3xuBG1/B+IA==", - "requires": { - "es6-promise": "3.2.1", - "mongodb-core": "2.1.20", - "readable-stream": "2.2.7" - }, - "dependencies": { - "es6-promise": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.2.1.tgz", - "integrity": "sha1-7FYjOGgDKQkgcXDDlEjiREndH8Q=" }, "process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=", + "dev": true }, "readable-stream": { "version": "2.2.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.7.tgz", "integrity": "sha1-BwV6y+JGeyIELTb5jFrVBwVOlbE=", + "dev": true, "requires": { "buffer-shims": "~1.0.0", "core-util-is": "~1.0.0", @@ -22151,12 +22227,42 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "dev": true, "requires": { "safe-buffer": "~5.1.0" } } } }, + "mongodb": { + "version": "3.5.6", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.6.tgz", + "integrity": "sha512-sh3q3GLDLT4QmoDLamxtAECwC3RGjq+oNuK1ENV8+tnipIavss6sMYt77hpygqlMOCt0Sla5cl7H4SKCVBCGEg==", + "requires": { + "bl": "^2.2.0", + "bson": "^1.1.4", + "denque": "^1.4.1", + "require_optional": "^1.0.1", + "safe-buffer": "^5.1.2", + "saslprep": "^1.0.0" + }, + "dependencies": { + "bl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz", + "integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==", + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "bson": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.4.tgz", + "integrity": "sha512-S/yKGU1syOMzO86+dGpg2qGoDL0zvzcb262G+gqEy6TgP6rt6z6qxSFX/8X6vLC91P7G7C3nLs0+bvDzmvBA3Q==" + } + } + }, "mongodb-core": { "version": "2.1.20", "resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-2.1.20.tgz", @@ -26888,6 +26994,15 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "saslprep": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, "sax": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", @@ -27656,6 +27771,15 @@ "integrity": "sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==", "dev": true }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + }, "spawn-sync": { "version": "1.0.15", "resolved": "https://registry.npmjs.org/spawn-sync/-/spawn-sync-1.0.15.tgz", diff --git a/server/methods/readMessages.js b/server/methods/readMessages.js index b33c61cd544d0..2f50f3706ffd0 100644 --- a/server/methods/readMessages.js +++ b/server/methods/readMessages.js @@ -3,6 +3,7 @@ import { check } from 'meteor/check'; import { callbacks } from '../../app/callbacks'; import { Subscriptions } from '../../app/models'; +import { Notification } from '../../app/notification-queue/server/NotificationQueue'; Meteor.methods({ readMessages(rid) { @@ -29,6 +30,8 @@ Meteor.methods({ Subscriptions.setAsReadByRoomIdAndUserId(rid, userId); + Notification.clearQueueForUser(userId); + Meteor.defer(() => { callbacks.run('afterReadMessages', rid, { userId, lastSeen: userSubscription.ls }); }); From 6bae623528ca3cac70432451aca1222f5fc19307 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 16:47:28 -0300 Subject: [PATCH 19/40] Fix unit tests --- app/models/server/models/Sessions.tests.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/models/server/models/Sessions.tests.js b/app/models/server/models/Sessions.tests.js index 5e9a49589bf47..b033284946c2b 100644 --- a/app/models/server/models/Sessions.tests.js +++ b/app/models/server/models/Sessions.tests.js @@ -245,21 +245,23 @@ describe('Sessions Aggregates', () => { after(() => { mongoUnit.stop(); }); } - before(function() { - return MongoClient.connect(process.env.MONGO_URL) - .then((client) => { db = client.db('test'); }); - }); + before(async () => { + const client = await MongoClient.connect(process.env.MONGO_URL); + db = client.db('test'); + + after(() => { + client.close(); + }); + + await db.dropDatabase(); - before(() => db.dropDatabase().then(() => { const sessions = db.collection('sessions'); const sessions_dates = db.collection('sessions_dates'); return Promise.all([ sessions.insertMany(DATA.sessions), sessions_dates.insertMany(DATA.sessions_dates), ]); - })); - - after(() => { db.close(); }); + }); it('should have sessions_dates data saved', () => { const collection = db.collection('sessions_dates'); From e82eccbf6a77bc9fadf2a7c5c36e404c7c916f54 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 16:59:14 -0300 Subject: [PATCH 20/40] Remove unecessary code/comments/setting --- app/lib/server/functions/notifications/email.js | 6 ------ app/lib/server/functions/notifications/mobile.js | 10 ---------- app/lib/server/lib/sendNotificationsOnMessage.js | 2 -- app/lib/server/startup/settings.js | 6 ------ packages/rocketchat-i18n/i18n/en.i18n.json | 2 -- 5 files changed, 26 deletions(-) diff --git a/app/lib/server/functions/notifications/email.js b/app/lib/server/functions/notifications/email.js index 2c42c65f0986b..1640977bd0ed3 100644 --- a/app/lib/server/functions/notifications/email.js +++ b/app/lib/server/functions/notifications/email.js @@ -166,7 +166,6 @@ export function sendEmail({ message, user, subscription, room, emailAddress, has export function shouldNotifyEmail({ disableAllMessageNotifications, - // statusConnection, emailNotifications, isHighlighted, hasMentionToUser, @@ -179,11 +178,6 @@ export function shouldNotifyEmail({ return false; } - // // use connected (don't need to send him an email) - // if (statusConnection === 'online') { - // return false; - // } - // user/room preference to nothing if (emailNotifications === 'nothing') { return false; diff --git a/app/lib/server/functions/notifications/mobile.js b/app/lib/server/functions/notifications/mobile.js index 15f793c25c7e9..0587b89ef4970 100644 --- a/app/lib/server/functions/notifications/mobile.js +++ b/app/lib/server/functions/notifications/mobile.js @@ -8,11 +8,6 @@ import { PushNotification } from '../../../../push-notifications/server'; const CATEGORY_MESSAGE = 'MESSAGE'; const CATEGORY_MESSAGE_NOREPLY = 'MESSAGE_NOREPLY'; -// let alwaysNotifyMobileBoolean; -// settings.get('Notifications_Always_Notify_Mobile', (key, value) => { -// alwaysNotifyMobileBoolean = value; -// }); - let SubscriptionRaw; Meteor.startup(() => { SubscriptionRaw = Subscriptions.model.rawCollection(); @@ -85,7 +80,6 @@ export function shouldNotifyMobile({ isHighlighted, hasMentionToUser, hasReplyToThread, - // statusConnection, roomType, }) { if (disableAllMessageNotifications && mobilePushNotifications == null && !isHighlighted && !hasMentionToUser && !hasReplyToThread) { @@ -96,10 +90,6 @@ export function shouldNotifyMobile({ return false; } - // if (!alwaysNotifyMobileBoolean && statusConnection === 'online') { - // return false; - // } - if (!mobilePushNotifications) { if (settings.get('Accounts_Default_User_Preferences_mobileNotifications') === 'all') { return true; diff --git a/app/lib/server/lib/sendNotificationsOnMessage.js b/app/lib/server/lib/sendNotificationsOnMessage.js index 05385fc67c1e9..a3984227854f8 100644 --- a/app/lib/server/lib/sendNotificationsOnMessage.js +++ b/app/lib/server/lib/sendNotificationsOnMessage.js @@ -125,7 +125,6 @@ export const sendNotification = async ({ isHighlighted, hasMentionToUser, hasReplyToThread, - statusConnection: receiver.statusConnection, roomType, })) { queueItems.push({ @@ -144,7 +143,6 @@ export const sendNotification = async ({ if (receiver.emails && shouldNotifyEmail({ disableAllMessageNotifications, - statusConnection: receiver.statusConnection, emailNotifications, isHighlighted, hasMentionToUser, diff --git a/app/lib/server/startup/settings.js b/app/lib/server/startup/settings.js index 33bf91e477afa..cfc7d2498bba4 100644 --- a/app/lib/server/startup/settings.js +++ b/app/lib/server/startup/settings.js @@ -925,12 +925,6 @@ settings.addGroup('General', function() { public: true, i18nDescription: 'Notifications_Max_Room_Members_Description', }); - - this.add('Notifications_Always_Notify_Mobile', false, { - type: 'boolean', - public: true, - i18nDescription: 'Notifications_Always_Notify_Mobile_Description', - }); }); this.section('REST API', function() { return this.add('API_User_Limit', 500, { diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index d0854c8776ab1..22c6e3d5acd8d 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -2507,8 +2507,6 @@ "Notification_RequireInteraction_Description": "Works only with Chrome browser versions > 50. Utilizes the parameter requireInteraction to show the desktop notification to indefinite until the user interacts with it.", "Notification_Mobile_Default_For": "Push Mobile Notifications For", "Notifications": "Notifications", - "Notifications_Always_Notify_Mobile": "Always notify mobile", - "Notifications_Always_Notify_Mobile_Description": "Choose to always notify mobile device regardless of presence status.", "Notifications_Duration": "Notifications Duration", "Notifications_Max_Room_Members": "Max Room Members Before Disabling All Message Notifications", "Notifications_Max_Room_Members_Description": "Max number of members in room when notifications for all messages gets disabled. Users can still change per room setting to receive all notifications on an individual basis. (0 to disable)", From d71b258953fafec7e4551ffb60108833db7a3938 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 17:19:20 -0300 Subject: [PATCH 21/40] Add indexes --- app/models/server/models/NotificationQueue.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/server/models/NotificationQueue.js b/app/models/server/models/NotificationQueue.js index b54eeadc9a85b..512e5fa313cdc 100644 --- a/app/models/server/models/NotificationQueue.js +++ b/app/models/server/models/NotificationQueue.js @@ -3,8 +3,10 @@ import { Base } from './_Base'; export class NotificationQueue extends Base { constructor() { super('notification_queue'); - this.tryEnsureIndex({ sid: 1 }); - // this.tryEnsureIndex({ 'room._id': 1, date: 1 }, { unique: true }); + this.tryEnsureIndex({ uid: 1 }); + this.tryEnsureIndex({ ts: 1 }); + this.tryEnsureIndex({ schedule: 1 }, { sparse: true }); + this.tryEnsureIndex({ sending: 1 }, { sparse: true }); } } From 9f16605b193d5c53913f71ebf18cb8cb258ff9a9 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 17:33:47 -0300 Subject: [PATCH 22/40] Init removal of _raix_push_notifications --- app/push/server/index.js | 2 +- app/push/server/push.js | 137 +++++------------------- app/statistics/server/lib/statistics.js | 4 +- server/startup/migrations/v181.js | 3 +- 4 files changed, 28 insertions(+), 118 deletions(-) diff --git a/app/push/server/index.js b/app/push/server/index.js index 35f25a9251d96..32ac42bd31776 100644 --- a/app/push/server/index.js +++ b/app/push/server/index.js @@ -1,3 +1,3 @@ import './methods'; -export { Push, appTokensCollection, notificationsCollection } from './push'; +export { Push, appTokensCollection } from './push'; diff --git a/app/push/server/push.js b/app/push/server/push.js index 13948830d7adc..b0547a1bdc32b 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -10,14 +10,9 @@ import { sendGCM } from './gcm'; import { logger, LoggerManager } from './logger'; export const _matchToken = Match.OneOf({ apn: String }, { gcm: String }); -export const notificationsCollection = new Mongo.Collection('_raix_push_notifications'); export const appTokensCollection = new Mongo.Collection('_raix_push_app_tokens'); appTokensCollection._ensureIndex({ userId: 1 }); -notificationsCollection._ensureIndex({ createdAt: 1 }); -notificationsCollection._ensureIndex({ sent: 1 }); -notificationsCollection._ensureIndex({ sending: 1 }); -notificationsCollection._ensureIndex({ delayUntil: 1 }); export class PushClass { options = {} @@ -53,116 +48,28 @@ export class PushClass { if (this.options.apn) { initAPN({ options: this.options, _removeToken: this._removeToken, absoluteUrl: Meteor.absoluteUrl() }); } + } - // This interval will allow only one notification to be sent at a time, it - // will check for new notifications at every `options.sendInterval` - // (default interval is 15000 ms) - // - // It looks in notifications collection to see if theres any pending - // notifications, if so it will try to reserve the pending notification. - // If successfully reserved the send is started. - // - // If notification.query is type string, it's assumed to be a json string - // version of the query selector. Making it able to carry `$` properties in - // the mongo collection. - // - // Pr. default notifications are removed from the collection after send have - // completed. Setting `options.keepNotifications` will update and keep the - // notification eg. if needed for historical reasons. - // - // After the send have completed a "send" event will be emitted with a - // status object containing notification id and the send result object. - // - let isSendingNotification = false; - - const sendNotification = (notification) => { - logger.debug('Sending notification', notification); - - // Reserve notification - const now = Date.now(); - const timeoutAt = now + this.options.sendTimeout; - const reserved = notificationsCollection.update({ - _id: notification._id, - sent: false, // xxx: need to make sure this is set on create - sending: { $lt: now }, - }, { - $set: { - sending: timeoutAt, - }, - }); - - // Make sure we only handle notifications reserved by this instance - if (reserved) { - // Check if query is set and is type String - if (notification.query && notification.query === String(notification.query)) { - try { - // The query is in string json format - we need to parse it - notification.query = JSON.parse(notification.query); - } catch (err) { - // Did the user tamper with this?? - throw new Error(`Error while parsing query string, Error: ${ err.message }`); - } - } - - // Send the notification - const result = this.serverSend(notification, this.options); - - if (!this.options.keepNotifications) { - // Pr. Default we will remove notifications - notificationsCollection.remove({ _id: notification._id }); - } else { - // Update the notification - notificationsCollection.update({ _id: notification._id }, { - $set: { - // Mark as sent - sent: true, - // Set the sent date - sentAt: new Date(), - // Count - count: result, - // Not being sent anymore - sending: 0, - }, - }); - } - } - }; - - this.sendWorker(() => { - if (isSendingNotification) { - return; - } + // If notification.query is type string, it's assumed to be a json string + // version of the query selector. Making it able to carry `$` properties in + // the mongo collection. + sendNotification(notification) { + logger.debug('Sending notification', notification); + // TODO: change to use token directly + // Check if query is set and is type String + if (notification.query && notification.query === String(notification.query)) { try { - // Set send fence - isSendingNotification = true; - - const batchSize = this.options.sendBatchSize || 1; - - // Find notifications that are not being or already sent - notificationsCollection.find({ - sent: false, - sending: { $lt: Date.now() }, - $or: [ - { delayUntil: { $exists: false } }, - { delayUntil: { $lte: new Date() } }, - ], - }, { - sort: { createdAt: 1 }, - limit: batchSize, - }).forEach((notification) => { - try { - sendNotification(notification); - } catch (error) { - logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); - logger.debug(error.stack); - } - }); - } finally { - // Remove the send fence - isSendingNotification = false; + // The query is in string json format - we need to parse it + notification.query = JSON.parse(notification.query); + } catch (err) { + // Did the user tamper with this?? + throw new Error(`Error while parsing query string, Error: ${ err.message }`); } - }, this.options.sendInterval || 15000); // Default every 15th sec + } + + // Send the notification + return this.serverSend(notification, this.options); } sendWorker(task, interval) { @@ -438,8 +345,12 @@ export class PushClass { // Validate the notification this._validateDocument(notification); - // Try to add the notification to send, we return an id to keep track - return notificationsCollection.insert(notification); + try { + this.sendNotification(notification); + } catch (error) { + logger.debug(`Could not send notification id: "${ notification._id }", Error: ${ error.message }`); + logger.debug(error.stack); + } } } diff --git a/app/statistics/server/lib/statistics.js b/app/statistics/server/lib/statistics.js index 141a79b9e49f2..cec42680bc97c 100644 --- a/app/statistics/server/lib/statistics.js +++ b/app/statistics/server/lib/statistics.js @@ -4,7 +4,6 @@ import _ from 'underscore'; import { Meteor } from 'meteor/meteor'; import { InstanceStatus } from 'meteor/konecty:multiple-instances-status'; -import { notificationsCollection } from '../../../push/server'; import { Sessions, Settings, @@ -22,6 +21,7 @@ import { Info, getMongoInfo } from '../../../utils/server'; import { Migrations } from '../../../migrations/server'; import { Apps } from '../../../apps/server'; import { getStatistics as federationGetStatistics } from '../../../federation/server/functions/dashboard'; +import { Notification } from '../../../notification-queue/server/NotificationQueue'; const wizardFields = [ 'Organization_Type', @@ -166,7 +166,7 @@ export const statistics = { totalWithScriptEnabled: integrations.filter((integration) => integration.scriptEnabled === true).length, }; - statistics.pushQueue = notificationsCollection.find().count(); + statistics.pushQueue = Promise.await(Notification.collection.estimatedDocumentCount()); return statistics; }, diff --git a/server/startup/migrations/v181.js b/server/startup/migrations/v181.js index 8f285d7293842..9ffb972cb60ce 100644 --- a/server/startup/migrations/v181.js +++ b/server/startup/migrations/v181.js @@ -1,4 +1,3 @@ -import { notificationsCollection } from '../../../app/push/server'; import { Migrations } from '../../../app/migrations/server'; import { Settings } from '../../../app/models/server'; @@ -12,6 +11,6 @@ Migrations.add({ date.setHours(date.getHours() - 2); // 2 hours ago; // Remove all records older than 2h - notificationsCollection.rawCollection().removeMany({ createdAt: { $lt: date } }); + // notificationsCollection.rawCollection().removeMany({ createdAt: { $lt: date } }); }, }); From 1a1132d5ef9e859266fb3a73156bd958755ca3c2 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 17:41:03 -0300 Subject: [PATCH 23/40] Use userId directly instead of query --- .../server/functions/notifications/mobile.js | 4 +- .../server/lib/PushNotification.js | 4 +- app/push/server/push.js | 50 +++++-------------- server/lib/pushConfig.js | 4 +- 4 files changed, 16 insertions(+), 46 deletions(-) diff --git a/app/lib/server/functions/notifications/mobile.js b/app/lib/server/functions/notifications/mobile.js index 0587b89ef4970..7a6bc3bf845f4 100644 --- a/app/lib/server/functions/notifications/mobile.js +++ b/app/lib/server/functions/notifications/mobile.js @@ -62,9 +62,7 @@ export async function getPushData({ room, message, userId, receiverUsername, sen username, message: settings.get('Push_show_message') ? notificationMessage : ' ', badge: await getBadgeCount(userId), - usersTo: { - userId, - }, + userId, category: enableNotificationReplyButton(room, receiverUsername) ? CATEGORY_MESSAGE : CATEGORY_MESSAGE_NOREPLY, }; } diff --git a/app/push-notifications/server/lib/PushNotification.js b/app/push-notifications/server/lib/PushNotification.js index 7a91478698eb8..89d51f6b274f3 100644 --- a/app/push-notifications/server/lib/PushNotification.js +++ b/app/push-notifications/server/lib/PushNotification.js @@ -20,7 +20,7 @@ export class PushNotification { return hash; } - send({ roomName, roomId, username, message, usersTo, payload, badge = 1, category }) { + send({ roomName, roomId, username, message, userId, payload, badge = 1, category }) { let title; if (roomName && roomName !== '') { title = `${ roomName }`; @@ -36,7 +36,7 @@ export class PushNotification { title, text: message, payload, - query: usersTo, + userId, notId: this.getNotificationId(roomId), gcm: { style: 'inbox', diff --git a/app/push/server/push.js b/app/push/server/push.js index b0547a1bdc32b..53806edf2d097 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -50,28 +50,6 @@ export class PushClass { } } - // If notification.query is type string, it's assumed to be a json string - // version of the query selector. Making it able to carry `$` properties in - // the mongo collection. - sendNotification(notification) { - logger.debug('Sending notification', notification); - - // TODO: change to use token directly - // Check if query is set and is type String - if (notification.query && notification.query === String(notification.query)) { - try { - // The query is in string json format - we need to parse it - notification.query = JSON.parse(notification.query); - } catch (err) { - // Did the user tamper with this?? - throw new Error(`Error while parsing query string, Error: ${ err.message }`); - } - } - - // Send the notification - return this.serverSend(notification, this.options); - } - sendWorker(task, interval) { logger.debug(`Send worker started, using interval: ${ interval }`); @@ -92,7 +70,7 @@ export class PushClass { appTokensCollection.rawCollection().updateMany({ token }, { $unset: { token: true } }); } - serverSendNative(app, notification, countApn, countGcm) { + sendNotificationNative(app, notification, countApn, countGcm) { logger.debug('send to token', app.token); notification.payload = notification.payload ? { ejson: EJSON.stringify(notification.payload) } : {}; @@ -162,7 +140,7 @@ export class PushClass { }); } - serverSendGateway(app, notification, countApn, countGcm) { + sendNotificationGateway(app, notification, countApn, countGcm) { for (const gateway of this.options.gateways) { logger.debug('send to token', app.token); @@ -179,7 +157,9 @@ export class PushClass { } } - serverSend(notification = { badge: 0 }) { + sendNotification(notification = { badge: 0 }) { + logger.debug('Sending notification', notification); + const countApn = []; const countGcm = []; @@ -196,27 +176,21 @@ export class PushClass { logger.debug(`send message "${ notification.title }" via query`, notification.query); const query = { - $and: [notification.query, { - $or: [{ - 'token.apn': { - $exists: true, - }, - }, { - 'token.gcm': { - $exists: true, - }, - }], - }], + userId: notification.userId, + $or: [ + { 'token.apn': { $exists: true } }, + { 'token.gcm': { $exists: true } }, + ], }; appTokensCollection.find(query).forEach((app) => { logger.debug('send to token', app.token); if (this.options.gateways) { - return this.serverSendGateway(app, notification, countApn, countGcm); + return this.sendNotificationGateway(app, notification, countApn, countGcm); } - return this.serverSendNative(app, notification, countApn, countGcm); + return this.sendNotificationNative(app, notification, countApn, countGcm); }); if (LoggerManager.logLevel === 2) { diff --git a/server/lib/pushConfig.js b/server/lib/pushConfig.js index e1828bf2cfe80..441397a383a8a 100644 --- a/server/lib/pushConfig.js +++ b/server/lib/pushConfig.js @@ -61,9 +61,7 @@ Meteor.methods({ text: `@${ user.username }:\n${ TAPi18n.__('This_is_a_push_test_messsage') }`, }, sound: 'default', - query: { - userId: user._id, - }, + userId: user._id, }); return { From cff3ed2eac92ef6dbd057a4f0f324513e8597f8c Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 23:15:35 -0300 Subject: [PATCH 24/40] Handle APN errors correctly --- app/push/server/apn.js | 29 ++++++++++++++--------------- app/push/server/push.js | 8 ++++---- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/app/push/server/apn.js b/app/push/server/apn.js index 54171e106f1bf..f661da45cda24 100644 --- a/app/push/server/apn.js +++ b/app/push/server/apn.js @@ -4,7 +4,7 @@ import { logger } from './logger'; let apnConnection; -export const sendAPN = (userToken, notification) => { +export const sendAPN = ({ userToken, notification, _removeToken }) => { if (typeof notification.apn === 'object') { notification = Object.assign({}, notification, notification.apn); } @@ -44,10 +44,21 @@ export const sendAPN = (userToken, notification) => { note.topic = notification.topic; note.mutableContent = 1; - apnConnection.send(note, userToken); + apnConnection.send(note, userToken).then((response) => { + response.failed.forEach((failure) => { + logger.debug(`Got error code ${ failure.status } for token ${ userToken }`); + + if (['400', '410'].includes(failure.status)) { + logger.debug(`Removing token ${ userToken }`); + _removeToken({ + apn: userToken, + }); + } + }); + }); }; -export const initAPN = ({ options, _removeToken, absoluteUrl }) => { +export const initAPN = ({ options, absoluteUrl }) => { logger.debug('APN configured'); // Allow production to be a general option for push notifications @@ -101,16 +112,4 @@ export const initAPN = ({ options, _removeToken, absoluteUrl }) => { // Rig apn connection apnConnection = new apn.Provider(options.apn); - - // Listen to transmission errors - should handle the same way as feedback. - apnConnection.on('transmissionError', (errCode, notification/* , recipient*/) => { - logger.debug('Got error code %d for token %s', errCode, notification.token); - - if ([2, 5, 8].indexOf(errCode) >= 0) { - // Invalid token errors... - _removeToken({ - apn: notification.token, - }); - } - }); }; diff --git a/app/push/server/push.js b/app/push/server/push.js index 13948830d7adc..381ecbf497b32 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -51,7 +51,7 @@ export class PushClass { logger.debug('Configure', this.options); if (this.options.apn) { - initAPN({ options: this.options, _removeToken: this._removeToken, absoluteUrl: Meteor.absoluteUrl() }); + initAPN({ options: this.options, absoluteUrl: Meteor.absoluteUrl() }); } // This interval will allow only one notification to be sent at a time, it @@ -182,7 +182,7 @@ export class PushClass { } _removeToken(token) { - appTokensCollection.rawCollection().updateMany({ token }, { $unset: { token: true } }); + appTokensCollection.rawCollection().deleteOne({ token }); } serverSendNative(app, notification, countApn, countGcm) { @@ -195,7 +195,7 @@ export class PushClass { // Send to APN if (this.options.apn) { notification.topic = app.appName; - sendAPN(app.token.apn, notification); + sendAPN({ userToken: app.token.apn, notification, _removeToken: this._removeToken }); } } else if (app.token.gcm) { countGcm.push(app._id); @@ -228,7 +228,7 @@ export class PushClass { return HTTP.post(`${ gateway }/push/${ service }/send`, data, (error, response) => { if (response && response.statusCode === 406) { - console.log('removing push token', token); + logger.info('removing push token', token); appTokensCollection.remove({ $or: [{ 'token.apn': token, From a8cb11b2db64b546d989bcb8b2c0e51e6f5572b6 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 23:38:21 -0300 Subject: [PATCH 25/40] Remove unused push settings --- app/lib/server/startup/settings.js | 28 +------------------ packages/rocketchat-i18n/i18n/en.i18n.json | 3 -- server/lib/pushConfig.js | 2 -- server/startup/migrations/index.js | 2 +- .../startup/migrations/{v181.js => v185.js} | 11 ++++---- 5 files changed, 8 insertions(+), 38 deletions(-) rename server/startup/migrations/{v181.js => v185.js} (50%) diff --git a/app/lib/server/startup/settings.js b/app/lib/server/startup/settings.js index cfc7d2498bba4..afa111d0d7e8d 100644 --- a/app/lib/server/startup/settings.js +++ b/app/lib/server/startup/settings.js @@ -1177,33 +1177,7 @@ settings.addGroup('Push', function() { public: true, alert: 'Push_Setting_Requires_Restart_Alert', }); - this.add('Push_debug', false, { - type: 'boolean', - public: true, - alert: 'Push_Setting_Requires_Restart_Alert', - enableQuery: { - _id: 'Push_enable', - value: true, - }, - }); - this.add('Push_send_interval', 2000, { - type: 'int', - public: true, - alert: 'Push_Setting_Requires_Restart_Alert', - enableQuery: { - _id: 'Push_enable', - value: true, - }, - }); - this.add('Push_send_batch_size', 100, { - type: 'int', - public: true, - alert: 'Push_Setting_Requires_Restart_Alert', - enableQuery: { - _id: 'Push_enable', - value: true, - }, - }); + this.add('Push_enable_gateway', true, { type: 'boolean', alert: 'Push_Setting_Requires_Restart_Alert', diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index 22c6e3d5acd8d..a9deff31821ee 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -2724,9 +2724,6 @@ "Push_apn_dev_passphrase": "APN Dev Passphrase", "Push_apn_key": "APN Key", "Push_apn_passphrase": "APN Passphrase", - "Push_debug": "Debug", - "Push_send_interval": "Interval to check the queue for new push notifications", - "Push_send_batch_size": "Batch size to be processed every tick", "Push_enable": "Enable", "Push_enable_gateway": "Enable Gateway", "Push_gateway": "Gateway", diff --git a/server/lib/pushConfig.js b/server/lib/pushConfig.js index 441397a383a8a..eb59ff138681b 100644 --- a/server/lib/pushConfig.js +++ b/server/lib/pushConfig.js @@ -110,8 +110,6 @@ function configurePush() { apn, gcm, production: settings.get('Push_production'), - sendInterval: settings.get('Push_send_interval'), - sendBatchSize: settings.get('Push_send_batch_size'), gateways: settings.get('Push_enable_gateway') === true ? settings.get('Push_gateway').split('\n') : undefined, uniqueId: settings.get('uniqueID'), getAuthorization() { diff --git a/server/startup/migrations/index.js b/server/startup/migrations/index.js index 1085c8ea77df0..96e3b4cd92d9e 100644 --- a/server/startup/migrations/index.js +++ b/server/startup/migrations/index.js @@ -178,8 +178,8 @@ import './v177'; import './v178'; import './v179'; import './v180'; -import './v181'; import './v182'; import './v183'; import './v184'; +import './v185'; import './xrun'; diff --git a/server/startup/migrations/v181.js b/server/startup/migrations/v185.js similarity index 50% rename from server/startup/migrations/v181.js rename to server/startup/migrations/v185.js index 9ffb972cb60ce..b5b13a614e9f2 100644 --- a/server/startup/migrations/v181.js +++ b/server/startup/migrations/v185.js @@ -2,13 +2,14 @@ import { Migrations } from '../../../app/migrations/server'; import { Settings } from '../../../app/models/server'; Migrations.add({ - version: 181, + version: 185, async up() { - Settings.update({ _id: 'Push_send_interval', value: 5000 }, { $set: { value: 2000 } }); - Settings.update({ _id: 'Push_send_batch_size', value: 10 }, { $set: { value: 100 } }); + Settings.remove({ _id: 'Push_send_interval' }); + Settings.remove({ _id: 'Push_send_batch_size' }); + Settings.remove({ _id: 'Push_debug' }); - const date = new Date(); - date.setHours(date.getHours() - 2); // 2 hours ago; + // const date = new Date(); + // date.setHours(date.getHours() - 2); // 2 hours ago; // Remove all records older than 2h // notificationsCollection.rawCollection().removeMany({ createdAt: { $lt: date } }); From 61eb21b00270b65a90ff40bcbd81e0c7e7934d33 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 23:42:25 -0300 Subject: [PATCH 26/40] Add TTL of 2h --- app/models/server/models/NotificationQueue.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/server/models/NotificationQueue.js b/app/models/server/models/NotificationQueue.js index 512e5fa313cdc..1210f5e73127f 100644 --- a/app/models/server/models/NotificationQueue.js +++ b/app/models/server/models/NotificationQueue.js @@ -4,7 +4,7 @@ export class NotificationQueue extends Base { constructor() { super('notification_queue'); this.tryEnsureIndex({ uid: 1 }); - this.tryEnsureIndex({ ts: 1 }); + this.tryEnsureIndex({ ts: 1 }, { expireAfterSeconds: 2 * 60 * 60 }); this.tryEnsureIndex({ schedule: 1 }, { sparse: true }); this.tryEnsureIndex({ sending: 1 }, { sparse: true }); } From 7f569b88ccf9152a747ec0a322b5fd64fb3ce871 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 23:51:16 -0300 Subject: [PATCH 27/40] Retry expired items on next query --- .../server/NotificationQueue.ts | 45 +++++++------------ tsconfig.json | 2 +- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index e384956539189..a875cd477dc81 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -25,9 +25,7 @@ interface INotificationItemPush { username: string; message: string; badge: number; - usersTo: { - userId: string; - }; + userId: string; category: string; }; } @@ -67,32 +65,13 @@ class NotificationClass { private maxBatchSize = 5; - private resetPendingInterval?: NodeJS.Timer; - - private resetPendingDelay = 5 * 60 * 1000; - initWorker(): void { this.running = true; this.executeWorkerLater(); - - this.resetPendingInterval = setInterval(() => { - const date = new Date(); - date.setMinutes(date.getMinutes() - 5); - this.collection.updateMany({ - sending: { $lt: date }, - }, { - $unset: { - sending: 1, - }, - }); - }, this.resetPendingDelay); } stopWorker(): void { this.running = false; - if (this.resetPendingInterval) { - clearInterval(this.resetPendingInterval); - } } executeWorkerLater(): void { @@ -159,15 +138,25 @@ class NotificationClass { } async getNextNotification(): Promise { + const now = new Date(); + const expired = new Date(); + expired.setMinutes(expired.getMinutes() - 5); + return (await this.collection.findOneAndUpdate({ - sending: { $exists: false }, - $or: [ - { schedule: { $exists: false } }, - { schedule: { $lte: new Date() } }, - ], + $and: [{ + $or: [ + { sending: { $exists: false } }, + { sending: { $lte: expired } }, + ], + }, { + $or: [ + { schedule: { $exists: false } }, + { schedule: { $lte: now } }, + ], + }], }, { $set: { - sending: new Date(), + sending: now, }, }, { sort: { diff --git a/tsconfig.json b/tsconfig.json index 84d64aaa3aaa0..3ac6fd928b035 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,7 +29,7 @@ }, "moduleResolution": "node", "resolveJsonModule": true, - "types": ["node", "mocha"], + "types": ["node"], "esModuleInterop": true, "preserveSymlinks": true, From f756cc8f44af3295f68c45a5957c41c079b12ab6 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 18 Apr 2020 23:52:17 -0300 Subject: [PATCH 28/40] Remove `Notifications_Always_Notify_Mobile` setting --- server/startup/migrations/v185.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/startup/migrations/v185.js b/server/startup/migrations/v185.js index b5b13a614e9f2..20304628f90e0 100644 --- a/server/startup/migrations/v185.js +++ b/server/startup/migrations/v185.js @@ -7,6 +7,7 @@ Migrations.add({ Settings.remove({ _id: 'Push_send_interval' }); Settings.remove({ _id: 'Push_send_batch_size' }); Settings.remove({ _id: 'Push_debug' }); + Settings.remove({ _id: 'Notifications_Always_Notify_Mobile' }); // const date = new Date(); // date.setHours(date.getHours() - 2); // 2 hours ago; From 2307807522cdec3b58f64409542ea1b760dc8b2a Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sun, 19 Apr 2020 00:58:58 -0300 Subject: [PATCH 29/40] Migrate and drop old collection --- .../server/NotificationQueue.ts | 6 +- app/push/server/push.js | 23 ++------ server/startup/migrations/v185.js | 55 +++++++++++++++++-- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index a875cd477dc81..ab3ed1898381b 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -1,3 +1,4 @@ +import { Meteor } from 'meteor/meteor'; import { Collection, ObjectId } from 'mongodb'; import { NotificationQueue, Users } from '../../models/server/raw'; @@ -224,4 +225,7 @@ class NotificationClass { } export const Notification = new NotificationClass(); -Notification.initWorker(); + +Meteor.startup(() => { + Notification.initWorker(); +}); diff --git a/app/push/server/push.js b/app/push/server/push.js index 9c43a853ac34d..69a094ddcc674 100644 --- a/app/push/server/push.js +++ b/app/push/server/push.js @@ -173,7 +173,7 @@ export class PushClass { throw new Error('Push.send: option "text" not a string'); } - logger.debug(`send message "${ notification.title }" via query`, notification.query); + logger.debug(`send message "${ notification.title }" to userId`, notification.userId); const query = { userId: notification.userId, @@ -257,23 +257,15 @@ export class PushClass { notId: Match.Optional(Match.Integer), }), android_channel_id: Match.Optional(String), - query: Match.Optional(String), - token: Match.Optional(_matchToken), - tokens: Match.Optional([_matchToken]), + userId: String, payload: Match.Optional(Object), delayUntil: Match.Optional(Date), createdAt: Date, createdBy: Match.OneOf(String, null), }); - // Make sure a token selector or query have been set - if (!notification.token && !notification.tokens && !notification.query) { - throw new Error('No token selector or query found'); - } - - // If tokens array is set it should not be empty - if (notification.tokens && !notification.tokens.length) { - throw new Error('No tokens in array'); + if (!notification.userId) { + throw new Error('No userId found'); } } @@ -290,7 +282,7 @@ export class PushClass { createdBy: currentUser, sent: false, sending: 0, - }, _.pick(options, 'from', 'title', 'text')); + }, _.pick(options, 'from', 'title', 'text', 'userId')); // Add extra Object.assign(notification, _.pick(options, 'payload', 'badge', 'sound', 'notId', 'delayUntil', 'android_channel_id')); @@ -303,11 +295,6 @@ export class PushClass { notification.gcm = _.pick(options.gcm, 'image', 'style', 'summaryText', 'picture', 'from', 'title', 'text', 'badge', 'sound', 'notId', 'actions', 'android_channel_id'); } - // Set one token selector, this can be token, array of tokens or query - if (options.query) { - notification.query = JSON.stringify(options.query); - } - if (options.contentAvailable != null) { notification.contentAvailable = options.contentAvailable; } diff --git a/server/startup/migrations/v185.js b/server/startup/migrations/v185.js index 20304628f90e0..e6c5be7be911c 100644 --- a/server/startup/migrations/v185.js +++ b/server/startup/migrations/v185.js @@ -1,5 +1,40 @@ +import { Mongo } from 'meteor/mongo'; + import { Migrations } from '../../../app/migrations/server'; import { Settings } from '../../../app/models/server'; +import { Notification } from '../../../app/notification-queue/server/NotificationQueue'; + +function convertNotification(notification) { + try { + const { userId } = JSON.parse(notification.query); + const username = notification.payload.sender?.username; + const roomName = notification.title !== username ? notification.title : ''; + + const message = roomName === '' ? notification.text : notification.text.replace(`${ username }: `, ''); + + return { + _id: notification._id, + uid: userId, + rid: notification.payload.rid, + ts: notification.createdAt, + items: [{ + type: 'push', + data: { + roomId: notification.payload.rid, + payload: notification.payload, + roomName, + username, + message, + badge: notification.badge, + userId, + category: notification.apn?.category, + }, + }], + }; + } catch (e) { + // + } +} Migrations.add({ version: 185, @@ -9,10 +44,22 @@ Migrations.add({ Settings.remove({ _id: 'Push_debug' }); Settings.remove({ _id: 'Notifications_Always_Notify_Mobile' }); - // const date = new Date(); - // date.setHours(date.getHours() - 2); // 2 hours ago; + const notificationsCollection = new Mongo.Collection('_raix_push_notifications'); + + const date = new Date(); + date.setHours(date.getHours() - 2); // 2 hours ago; + + const cursor = notificationsCollection.rawCollection().find({ + createdAt: { $gte: date }, + }); + + for await (const notification of cursor) { + const newNotification = convertNotification(notification); + if (newNotification) { + await Notification.collection.insertOne(newNotification); + } + } - // Remove all records older than 2h - // notificationsCollection.rawCollection().removeMany({ createdAt: { $lt: date } }); + notificationsCollection.rawCollection().drop(); }, }); From cf768d4cad6232757b74471d17bb7e5dac8d3cdb Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sun, 19 Apr 2020 01:06:03 -0300 Subject: [PATCH 30/40] Organize variables --- app/notification-queue/server/NotificationQueue.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index ab3ed1898381b..f6270471da510 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -62,9 +62,11 @@ class NotificationClass { private running = false; - private delay = 2000; + private cyclePause = 2000; - private maxBatchSize = 5; + private maxBatchSize = 100; + + private maxScheduleDelaySeconds = 120 initWorker(): void { this.running = true; @@ -80,7 +82,7 @@ class NotificationClass { return; } - setTimeout(this.worker.bind(this), this.delay); + setTimeout(this.worker.bind(this), this.cyclePause); } async worker(counter = 0): Promise { @@ -188,8 +190,7 @@ class NotificationClass { return; } - // TODO: Make it configurable - const delay = 120; + const delay = this.maxScheduleDelaySeconds; let schedule: Date | undefined; From fec51904f2d604dab04b20bb2e7284cfa713269f Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 20 Apr 2020 11:32:44 -0300 Subject: [PATCH 31/40] Fix lint --- app/statistics/server/lib/statistics.js | 1 - 1 file changed, 1 deletion(-) diff --git a/app/statistics/server/lib/statistics.js b/app/statistics/server/lib/statistics.js index 58233201bc136..cec42680bc97c 100644 --- a/app/statistics/server/lib/statistics.js +++ b/app/statistics/server/lib/statistics.js @@ -4,7 +4,6 @@ import _ from 'underscore'; import { Meteor } from 'meteor/meteor'; import { InstanceStatus } from 'meteor/konecty:multiple-instances-status'; -import { notificationsCollection } from '../../../push/server'; import { Sessions, Settings, From 2886c3e872e18111f871af962af35f6c9bdcca81 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Mon, 20 Apr 2020 18:19:41 -0300 Subject: [PATCH 32/40] Free disk space on CI --- .github/workflows/build_and_test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6d0c06c5f360b..2d74d00f0e178 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -250,6 +250,14 @@ jobs: steps: - uses: actions/checkout@v1 + - name: Free disk space + run: | + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + docker rmi $(docker image ls -aq) + df -h + - name: Cache node modules id: cache-nodemodules uses: actions/cache@v1 From b77588b26d64cfcca2eaefb7743369f3552e5910 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 00:30:27 -0300 Subject: [PATCH 33/40] General improvements --- .eslintrc | 3 +- .../server/functions/notifications/mobile.js | 9 -- .../server/lib/sendNotificationsOnMessage.js | 1 + app/models/server/raw/NotificationQueue.js | 5 - app/models/server/raw/NotificationQueue.ts | 77 ++++++++++ .../server/NotificationQueue.ts | 141 +++--------------- .../server/lib/PushNotification.js | 13 +- app/statistics/server/lib/statistics.js | 4 +- definition/INotification.ts | 46 ++++++ server/methods/readMessages.js | 8 +- 10 files changed, 165 insertions(+), 142 deletions(-) delete mode 100644 app/models/server/raw/NotificationQueue.js create mode 100644 app/models/server/raw/NotificationQueue.ts create mode 100644 definition/INotification.ts diff --git a/.eslintrc b/.eslintrc index efef5b7d68d80..7bb7c4d7da9ca 100644 --- a/.eslintrc +++ b/.eslintrc @@ -90,7 +90,8 @@ "@typescript-eslint/interface-name-prefix": [ "error", "always" - ] + ], + "@typescript-eslint/explicit-function-return-type": "off" }, "env": { "browser": true, diff --git a/app/lib/server/functions/notifications/mobile.js b/app/lib/server/functions/notifications/mobile.js index 7a6bc3bf845f4..0d09f11eb836b 100644 --- a/app/lib/server/functions/notifications/mobile.js +++ b/app/lib/server/functions/notifications/mobile.js @@ -3,7 +3,6 @@ import { Meteor } from 'meteor/meteor'; import { settings } from '../../../../settings'; import { Subscriptions } from '../../../../models'; import { roomTypes } from '../../../../utils'; -import { PushNotification } from '../../../../push-notifications/server'; const CATEGORY_MESSAGE = 'MESSAGE'; const CATEGORY_MESSAGE_NOREPLY = 'MESSAGE_NOREPLY'; @@ -48,29 +47,21 @@ export async function getPushData({ room, message, userId, receiverUsername, sen } return { - roomId: message.rid, payload: { host: Meteor.absoluteUrl(), - rid: message.rid, sender: message.u, type: room.t, name: room.name, messageType: message.t, - messageId: message._id, }, roomName: settings.get('Push_show_username_room') && roomTypes.getConfig(room.t).isGroupChat(room) ? `#${ roomTypes.getRoomName(room.t, room) }` : '', username, message: settings.get('Push_show_message') ? notificationMessage : ' ', badge: await getBadgeCount(userId), - userId, category: enableNotificationReplyButton(room, receiverUsername) ? CATEGORY_MESSAGE : CATEGORY_MESSAGE_NOREPLY, }; } -export async function sendSinglePush({ room, message, userId, receiverUsername, senderUsername, senderName, notificationMessage }) { - PushNotification.send(getPushData({ room, message, userId, receiverUsername, senderUsername, senderName, notificationMessage })); -} - export function shouldNotifyMobile({ disableAllMessageNotifications, mobilePushNotifications, diff --git a/app/lib/server/lib/sendNotificationsOnMessage.js b/app/lib/server/lib/sendNotificationsOnMessage.js index a3984227854f8..7a6406cbfb868 100644 --- a/app/lib/server/lib/sendNotificationsOnMessage.js +++ b/app/lib/server/lib/sendNotificationsOnMessage.js @@ -168,6 +168,7 @@ export const sendNotification = async ({ uid: subscription.u._id, rid: room._id, sid: subscription._id, + mid: message._id, items: queueItems, }); } diff --git a/app/models/server/raw/NotificationQueue.js b/app/models/server/raw/NotificationQueue.js deleted file mode 100644 index 56403d2833822..0000000000000 --- a/app/models/server/raw/NotificationQueue.js +++ /dev/null @@ -1,5 +0,0 @@ -import { BaseRaw } from './BaseRaw'; - -export class NotificationQueueRaw extends BaseRaw { - -} diff --git a/app/models/server/raw/NotificationQueue.ts b/app/models/server/raw/NotificationQueue.ts new file mode 100644 index 0000000000000..90dce3ebba9f6 --- /dev/null +++ b/app/models/server/raw/NotificationQueue.ts @@ -0,0 +1,77 @@ +import { + Collection, + ObjectId, +} from 'mongodb'; + +import { BaseRaw } from './BaseRaw'; +import { INotification } from '../../../../definition/INotification'; + +export class NotificationQueueRaw extends BaseRaw { + public readonly col!: Collection; + + unsetSendingById(_id: string) { + return this.col.updateOne({ _id }, { + $unset: { + sending: 1, + }, + }); + } + + removeById(_id: string) { + return this.col.deleteOne({ _id }); + } + + clearScheduleByUserId(uid: string) { + return this.col.updateMany({ + uid, + schedule: { $exists: true }, + }, { + $unset: { + schedule: 1, + }, + }); + } + + async clearQueueByUserId(uid: string): Promise { + const op = await this.col.deleteMany({ + uid, + }); + + return op.deletedCount; + } + + async findNextInQueueOrExpired(expired: Date): Promise { + const now = new Date(); + + const result = await this.col.findOneAndUpdate({ + $and: [{ + $or: [ + { sending: { $exists: false } }, + { sending: { $lte: expired } }, + ], + }, { + $or: [ + { schedule: { $exists: false } }, + { schedule: { $lte: now } }, + ], + }], + }, { + $set: { + sending: now, + }, + }, { + sort: { + ts: 1, + }, + }); + + return result.value; + } + + insertOne(data: Omit) { + return this.col.insertOne({ + _id: new ObjectId().toHexString(), + ...data, + }); + } +} diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index f6270471da510..ddef5b7b10216 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -1,65 +1,11 @@ import { Meteor } from 'meteor/meteor'; -import { Collection, ObjectId } from 'mongodb'; +import { INotification, INotificationItemPush, INotificationItemEmail, NotificationItem } from '../../../definition/INotification'; import { NotificationQueue, Users } from '../../models/server/raw'; import { sendEmailFromData } from '../../lib/server/functions/notifications/email'; import { PushNotification } from '../../push-notifications/server'; -const UsersCollection: Collection = Users.col; - -interface INotificationItemPush { - type: 'push'; - data: { - roomId: string; - payload: { - host: string; - rid: string; - sender: { - _id: string; - username: string; - name?: string; - }; - type: string; - messageId: string; - }; - roomName: string; - username: string; - message: string; - badge: number; - userId: string; - category: string; - }; -} - -interface INotificationItemEmail { - type: 'email'; - data: { - to: string; - subject: string; - html: string; - data: { - room_path: string; - }; - from: string; - }; -} - -type NotificationItem = INotificationItemPush | INotificationItemEmail; - -interface INotification { - _id: string; - uid: string; - rid: string; - sid: string; - ts: Date; - schedule?: Date; - sending?: Date; - items: NotificationItem[]; -} - class NotificationClass { - public readonly collection: Collection = NotificationQueue.col - private running = false; private cyclePause = 2000; @@ -93,7 +39,13 @@ class NotificationClass { } // Once we start notifying the user we anticipate all the schedules - this.flushQueueForUser(notification.uid); + const flush = await NotificationQueue.clearScheduleByUserId(notification.uid); + + // start worker again it queue flushed + if (flush.modifiedCount) { + await NotificationQueue.unsetSendingById(notification._id); + return this.worker(counter); + } console.log('processing', notification._id); @@ -101,7 +53,7 @@ class NotificationClass { for (const item of notification.items) { switch (item.type) { case 'push': - this.push(item); + this.push(notification, item); break; case 'email': this.email(item); @@ -109,18 +61,10 @@ class NotificationClass { } } - this.collection.deleteOne({ - _id: notification._id, - }); + NotificationQueue.removeById(notification._id); } catch (e) { console.error(e); - this.collection.updateOne({ - _id: notification._id, - }, { - $unset: { - sending: 1, - }, - }); + await NotificationQueue.unsetSendingById(notification._id); } if (counter >= this.maxBatchSize) { @@ -129,57 +73,28 @@ class NotificationClass { this.worker(counter++); } - async flushQueueForUser(userId: string): Promise { - await this.collection.updateMany({ - uid: userId, - schedule: { $exists: true }, - }, { - $unset: { - schedule: 1, - }, - }); - } - - async getNextNotification(): Promise { - const now = new Date(); + getNextNotification(): Promise { const expired = new Date(); expired.setMinutes(expired.getMinutes() - 5); - return (await this.collection.findOneAndUpdate({ - $and: [{ - $or: [ - { sending: { $exists: false } }, - { sending: { $lte: expired } }, - ], - }, { - $or: [ - { schedule: { $exists: false } }, - { schedule: { $lte: now } }, - ], - }], - }, { - $set: { - sending: now, - }, - }, { - sort: { - ts: 1, - }, - })).value; + return NotificationQueue.findNextInQueueOrExpired(expired); } - push(item: INotificationItemPush): void { - PushNotification.send(item.data); + push({ uid, rid, mid }: INotification, item: INotificationItemPush): void { + PushNotification.send({ + rid, + uid, + mid, + ...item.data, + }); } email(item: INotificationItemEmail): void { sendEmailFromData(item.data); } - async scheduleItem({ uid, rid, sid, items }: {uid: string; rid: string; sid: string; items: NotificationItem[]}): Promise { - const user = await UsersCollection.findOne({ - _id: uid, - }, { + async scheduleItem({ uid, rid, sid, mid, items }: {uid: string; rid: string; sid: string; mid: string; items: NotificationItem[]}): Promise { + const user = await Users.findOneById(uid, { projection: { statusConnection: 1, _updatedAt: 1, @@ -205,24 +120,16 @@ class NotificationClass { } } - await this.collection.insertOne({ - _id: new ObjectId().toString(), + await NotificationQueue.insertOne({ uid, rid, sid, + mid, ts: new Date(), schedule, items, }); } - - async clearQueueForUser(userId: string): Promise { - const op = await this.collection.deleteMany({ - uid: userId, - }); - - return op.deletedCount; - } } export const Notification = new NotificationClass(); diff --git a/app/push-notifications/server/lib/PushNotification.js b/app/push-notifications/server/lib/PushNotification.js index 89d51f6b274f3..0089b3ff4ac40 100644 --- a/app/push-notifications/server/lib/PushNotification.js +++ b/app/push-notifications/server/lib/PushNotification.js @@ -20,7 +20,7 @@ export class PushNotification { return hash; } - send({ roomName, roomId, username, message, userId, payload, badge = 1, category }) { + send({ rid, uid, mid, roomName, username, message, payload, badge = 1, category }) { let title; if (roomName && roomName !== '') { title = `${ roomName }`; @@ -28,6 +28,7 @@ export class PushNotification { } else { title = `${ username }`; } + const config = { from: 'push', badge, @@ -35,9 +36,13 @@ export class PushNotification { priority: 10, title, text: message, - payload, - userId, - notId: this.getNotificationId(roomId), + payload: { + rid, + messageId: mid, + ...payload, + }, + userId: uid, + notId: this.getNotificationId(rid), gcm: { style: 'inbox', summaryText: '%n% new messages', diff --git a/app/statistics/server/lib/statistics.js b/app/statistics/server/lib/statistics.js index cec42680bc97c..06dcca34dbfff 100644 --- a/app/statistics/server/lib/statistics.js +++ b/app/statistics/server/lib/statistics.js @@ -21,7 +21,7 @@ import { Info, getMongoInfo } from '../../../utils/server'; import { Migrations } from '../../../migrations/server'; import { Apps } from '../../../apps/server'; import { getStatistics as federationGetStatistics } from '../../../federation/server/functions/dashboard'; -import { Notification } from '../../../notification-queue/server/NotificationQueue'; +import { NotificationQueue } from '../../../models/server/raw'; const wizardFields = [ 'Organization_Type', @@ -166,7 +166,7 @@ export const statistics = { totalWithScriptEnabled: integrations.filter((integration) => integration.scriptEnabled === true).length, }; - statistics.pushQueue = Promise.await(Notification.collection.estimatedDocumentCount()); + statistics.pushQueue = Promise.await(NotificationQueue.col.estimatedDocumentCount()); return statistics; }, diff --git a/definition/INotification.ts b/definition/INotification.ts new file mode 100644 index 0000000000000..46b3bf99dfeaa --- /dev/null +++ b/definition/INotification.ts @@ -0,0 +1,46 @@ +export interface INotificationItemPush { + type: 'push'; + data: { + payload: { + host: string; + sender: { + _id: string; + username: string; + name?: string; + }; + type: string; + }; + roomName: string; + username: string; + message: string; + badge: number; + category: string; + }; +} + +export interface INotificationItemEmail { + type: 'email'; + data: { + to: string; + subject: string; + html: string; + data: { + room_path: string; + }; + from: string; + }; +} + +export type NotificationItem = INotificationItemPush | INotificationItemEmail; + +export interface INotification { + _id: string; + uid: string; + rid: string; + sid: string; + mid: string; + ts: Date; + schedule?: Date; + sending?: Date; + items: NotificationItem[]; +} diff --git a/server/methods/readMessages.js b/server/methods/readMessages.js index 2f50f3706ffd0..08b394f8c51da 100644 --- a/server/methods/readMessages.js +++ b/server/methods/readMessages.js @@ -1,9 +1,9 @@ import { Meteor } from 'meteor/meteor'; import { check } from 'meteor/check'; -import { callbacks } from '../../app/callbacks'; -import { Subscriptions } from '../../app/models'; -import { Notification } from '../../app/notification-queue/server/NotificationQueue'; +import { callbacks } from '../../app/callbacks/server'; +import { Subscriptions } from '../../app/models/server'; +import { NotificationQueue } from '../../app/models/server/raw'; Meteor.methods({ readMessages(rid) { @@ -30,7 +30,7 @@ Meteor.methods({ Subscriptions.setAsReadByRoomIdAndUserId(rid, userId); - Notification.clearQueueForUser(userId); + NotificationQueue.clearQueueByUserId(userId); Meteor.defer(() => { callbacks.run('afterReadMessages', rid, { userId, lastSeen: userSubscription.ls }); From ea6556b9c416d7774ac05f31e0542775110ecf3a Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 00:45:58 -0300 Subject: [PATCH 34/40] Remove insert from BaseRaw --- app/models/server/raw/BaseRaw.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app/models/server/raw/BaseRaw.js b/app/models/server/raw/BaseRaw.js index 15a80bb37bb6a..614c196ee7bad 100644 --- a/app/models/server/raw/BaseRaw.js +++ b/app/models/server/raw/BaseRaw.js @@ -1,5 +1,3 @@ -import { ObjectId } from 'mongodb'; - export class BaseRaw { constructor(col) { this.col = col; @@ -21,13 +19,6 @@ export class BaseRaw { return this.col.find(...args); } - insert(docs, options) { - if (!Array.isArray(docs) && !docs._id) { - docs._id = new ObjectId().toString(); - } - return this.col.insert(docs, options); - } - update(...args) { return this.col.update(...args); } From e53ed51ee0d480d19b7125e03e5924b36c3619f9 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 01:00:51 -0300 Subject: [PATCH 35/40] Remove host from push items --- app/lib/server/functions/notifications/mobile.js | 1 - .../server/lib/PushNotification.js | 16 +++++++++------- definition/INotification.ts | 1 - 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/lib/server/functions/notifications/mobile.js b/app/lib/server/functions/notifications/mobile.js index 0d09f11eb836b..fdc5dda7b4df4 100644 --- a/app/lib/server/functions/notifications/mobile.js +++ b/app/lib/server/functions/notifications/mobile.js @@ -48,7 +48,6 @@ export async function getPushData({ room, message, userId, receiverUsername, sen return { payload: { - host: Meteor.absoluteUrl(), sender: message.u, type: room.t, name: room.name, diff --git a/app/push-notifications/server/lib/PushNotification.js b/app/push-notifications/server/lib/PushNotification.js index 0089b3ff4ac40..76847b6f4c3c5 100644 --- a/app/push-notifications/server/lib/PushNotification.js +++ b/app/push-notifications/server/lib/PushNotification.js @@ -1,7 +1,9 @@ +import { Meteor } from 'meteor/meteor'; + import { Push } from '../../../push/server'; -import { settings } from '../../../settings'; -import { metrics } from '../../../metrics'; -import { RocketChatAssets } from '../../../assets'; +import { settings } from '../../../settings/server'; +import { metrics } from '../../../metrics/server'; +import { RocketChatAssets } from '../../../assets/server'; export class PushNotification { getNotificationId(roomId) { @@ -20,7 +22,7 @@ export class PushNotification { return hash; } - send({ rid, uid, mid, roomName, username, message, payload, badge = 1, category }) { + send({ rid, uid: userId, mid: messageId, roomName, username, message, payload, badge = 1, category }) { let title; if (roomName && roomName !== '') { title = `${ roomName }`; @@ -37,15 +39,15 @@ export class PushNotification { title, text: message, payload: { + host: Meteor.absoluteUrl(), rid, - messageId: mid, + messageId, ...payload, }, - userId: uid, + userId, notId: this.getNotificationId(rid), gcm: { style: 'inbox', - summaryText: '%n% new messages', image: RocketChatAssets.getURL('Assets_favicon_192'), }, }; diff --git a/definition/INotification.ts b/definition/INotification.ts index 46b3bf99dfeaa..23640f1cbce8b 100644 --- a/definition/INotification.ts +++ b/definition/INotification.ts @@ -2,7 +2,6 @@ export interface INotificationItemPush { type: 'push'; data: { payload: { - host: string; sender: { _id: string; username: string; From dca2ebbaa39b75e09b5e6efecfca58bd4b4d8868 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 01:16:51 -0300 Subject: [PATCH 36/40] Rename migration --- server/startup/migrations/index.js | 2 +- server/startup/migrations/{v185.js => v187.js} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename server/startup/migrations/{v185.js => v187.js} (99%) diff --git a/server/startup/migrations/index.js b/server/startup/migrations/index.js index 96e3b4cd92d9e..b60af6d82bf70 100644 --- a/server/startup/migrations/index.js +++ b/server/startup/migrations/index.js @@ -181,5 +181,5 @@ import './v180'; import './v182'; import './v183'; import './v184'; -import './v185'; +import './v187'; import './xrun'; diff --git a/server/startup/migrations/v185.js b/server/startup/migrations/v187.js similarity index 99% rename from server/startup/migrations/v185.js rename to server/startup/migrations/v187.js index e6c5be7be911c..f30d0fd4fb408 100644 --- a/server/startup/migrations/v185.js +++ b/server/startup/migrations/v187.js @@ -37,7 +37,7 @@ function convertNotification(notification) { } Migrations.add({ - version: 185, + version: 187, async up() { Settings.remove({ _id: 'Push_send_interval' }); Settings.remove({ _id: 'Push_send_batch_size' }); From 1e3ca2b0d807630e4e845ddf5c40c46c9f9b148f Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 01:45:33 -0300 Subject: [PATCH 37/40] Use Fibers on migration --- server/startup/migrations/v187.js | 43 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/server/startup/migrations/v187.js b/server/startup/migrations/v187.js index f30d0fd4fb408..6b430b8f7629e 100644 --- a/server/startup/migrations/v187.js +++ b/server/startup/migrations/v187.js @@ -2,7 +2,7 @@ import { Mongo } from 'meteor/mongo'; import { Migrations } from '../../../app/migrations/server'; import { Settings } from '../../../app/models/server'; -import { Notification } from '../../../app/notification-queue/server/NotificationQueue'; +import { NotificationQueue } from '../../../app/models/server/raw'; function convertNotification(notification) { try { @@ -16,17 +16,16 @@ function convertNotification(notification) { _id: notification._id, uid: userId, rid: notification.payload.rid, + mid: notification.payload.messageId, ts: notification.createdAt, items: [{ type: 'push', data: { - roomId: notification.payload.rid, payload: notification.payload, roomName, username, message, badge: notification.badge, - userId, category: notification.apn?.category, }, }], @@ -36,30 +35,32 @@ function convertNotification(notification) { } } +async function migrateNotifications() { + const notificationsCollection = new Mongo.Collection('_raix_push_notifications'); + + const date = new Date(); + date.setHours(date.getHours() - 2); // 2 hours ago; + + const cursor = notificationsCollection.rawCollection().find({ + createdAt: { $gte: date }, + }); + for await (const notification of cursor) { + const newNotification = convertNotification(notification); + if (newNotification) { + await NotificationQueue.insertOne(newNotification); + } + } + return notificationsCollection.rawCollection().drop(); +} + Migrations.add({ version: 187, - async up() { + up() { Settings.remove({ _id: 'Push_send_interval' }); Settings.remove({ _id: 'Push_send_batch_size' }); Settings.remove({ _id: 'Push_debug' }); Settings.remove({ _id: 'Notifications_Always_Notify_Mobile' }); - const notificationsCollection = new Mongo.Collection('_raix_push_notifications'); - - const date = new Date(); - date.setHours(date.getHours() - 2); // 2 hours ago; - - const cursor = notificationsCollection.rawCollection().find({ - createdAt: { $gte: date }, - }); - - for await (const notification of cursor) { - const newNotification = convertNotification(notification); - if (newNotification) { - await Notification.collection.insertOne(newNotification); - } - } - - notificationsCollection.rawCollection().drop(); + Promise.await(migrateNotifications()); }, }); From 47e96054a2a3429cf18f2d154268834d0df0a3eb Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 02:34:37 -0300 Subject: [PATCH 38/40] Remove sid --- app/lib/server/lib/sendNotificationsOnMessage.js | 1 - app/notification-queue/server/NotificationQueue.ts | 3 +-- definition/INotification.ts | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/lib/server/lib/sendNotificationsOnMessage.js b/app/lib/server/lib/sendNotificationsOnMessage.js index c842ea933e2a3..834eac239b0e7 100644 --- a/app/lib/server/lib/sendNotificationsOnMessage.js +++ b/app/lib/server/lib/sendNotificationsOnMessage.js @@ -167,7 +167,6 @@ export const sendNotification = async ({ Notification.scheduleItem({ uid: subscription.u._id, rid: room._id, - sid: subscription._id, mid: message._id, items: queueItems, }); diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index ddef5b7b10216..b583bbeac18f8 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -93,7 +93,7 @@ class NotificationClass { sendEmailFromData(item.data); } - async scheduleItem({ uid, rid, sid, mid, items }: {uid: string; rid: string; sid: string; mid: string; items: NotificationItem[]}): Promise { + async scheduleItem({ uid, rid, mid, items }: {uid: string; rid: string; mid: string; items: NotificationItem[]}): Promise { const user = await Users.findOneById(uid, { projection: { statusConnection: 1, @@ -123,7 +123,6 @@ class NotificationClass { await NotificationQueue.insertOne({ uid, rid, - sid, mid, ts: new Date(), schedule, diff --git a/definition/INotification.ts b/definition/INotification.ts index 23640f1cbce8b..6359f33dd612c 100644 --- a/definition/INotification.ts +++ b/definition/INotification.ts @@ -36,7 +36,6 @@ export interface INotification { _id: string; uid: string; rid: string; - sid: string; mid: string; ts: Date; schedule?: Date; From d2d1d6c57f0487ebfc900b083d99c3a734890719 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 02:34:52 -0300 Subject: [PATCH 39/40] Cofigured params via env vars --- app/notification-queue/server/NotificationQueue.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index b583bbeac18f8..76af8966e97bb 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -5,14 +5,20 @@ import { NotificationQueue, Users } from '../../models/server/raw'; import { sendEmailFromData } from '../../lib/server/functions/notifications/email'; import { PushNotification } from '../../push-notifications/server'; +const { + NOTIFICATIONS_WORKER_TIMEOUT = '2000', + NOTIFICATIONS_BATCH_SIZE = '100', + NOTIFICATIONS_SCHEDULE_DELAY = '120', +} = process.env; + class NotificationClass { private running = false; - private cyclePause = 2000; + private cyclePause = parseInt(NOTIFICATIONS_WORKER_TIMEOUT); - private maxBatchSize = 100; + private maxBatchSize = parseInt(NOTIFICATIONS_BATCH_SIZE); - private maxScheduleDelaySeconds = 120 + private maxScheduleDelaySeconds = parseInt(NOTIFICATIONS_SCHEDULE_DELAY); initWorker(): void { this.running = true; From 1a9ddb07f42fe5742bcb56e3af25bcc9483041e3 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 21 Apr 2020 02:39:00 -0300 Subject: [PATCH 40/40] Use Number --- app/notification-queue/server/NotificationQueue.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/notification-queue/server/NotificationQueue.ts b/app/notification-queue/server/NotificationQueue.ts index 76af8966e97bb..758294542826c 100644 --- a/app/notification-queue/server/NotificationQueue.ts +++ b/app/notification-queue/server/NotificationQueue.ts @@ -6,19 +6,19 @@ import { sendEmailFromData } from '../../lib/server/functions/notifications/emai import { PushNotification } from '../../push-notifications/server'; const { - NOTIFICATIONS_WORKER_TIMEOUT = '2000', - NOTIFICATIONS_BATCH_SIZE = '100', - NOTIFICATIONS_SCHEDULE_DELAY = '120', + NOTIFICATIONS_WORKER_TIMEOUT = 2000, + NOTIFICATIONS_BATCH_SIZE = 100, + NOTIFICATIONS_SCHEDULE_DELAY = 120, } = process.env; class NotificationClass { private running = false; - private cyclePause = parseInt(NOTIFICATIONS_WORKER_TIMEOUT); + private cyclePause = Number(NOTIFICATIONS_WORKER_TIMEOUT); - private maxBatchSize = parseInt(NOTIFICATIONS_BATCH_SIZE); + private maxBatchSize = Number(NOTIFICATIONS_BATCH_SIZE); - private maxScheduleDelaySeconds = parseInt(NOTIFICATIONS_SCHEDULE_DELAY); + private maxScheduleDelaySeconds = Number(NOTIFICATIONS_SCHEDULE_DELAY); initWorker(): void { this.running = true;