diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e4b9d207..d43f9d7f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,8 @@ name: CI on: [push, pull_request] env: - go-version: '1.17.x' + go-version: '1.17.2' # https://github.com/golang/go/issues/49366 + redis-version: '3.2.4' jobs: test: name: Test @@ -16,7 +17,7 @@ jobs: - name: Install Redis uses: zhulik/redis-action@v1.0.0 with: - redis version: '5' + redis version: ${{ env.redis-version }} - name: Install PostgreSQL uses: harmon758/postgresql-action@v1 diff --git a/.gitignore b/.gitignore index 6b5579d0b..86bba308f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.a *.so *~ +deploy fabric fabfile.py fabfile.pyc @@ -34,4 +35,4 @@ _testmain.go dist/ .envrc courier -_storage \ No newline at end of file +_storage diff --git a/CHANGELOG.md b/CHANGELOG.md index d8af3263b..d0ee8624b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,71 @@ +v7.1.14 +---------- + * Allow more active redis connections + * Support sending WA quick replies when we have attachments too + * Add support to receive button text from Twilio WhatsApp + +v7.1.13 +---------- + * Send db and redis stats to librato in backed heartbeat + * Include session_status in FCM payloads + +v7.1.12 +---------- + * Update to latest gocommon + * Add instagram handler + +v7.1.11 +---------- + * More bulk sql tweaks + +v7.1.10 +---------- + * Update to latest gocommon + +v7.1.9 +---------- + * Fix bulk status updates + +v7.1.8 +---------- + * Do more error wrapping when creating contacts and URNs + +v7.1.7 +---------- + * Use dbutil package from gocommon + * Add quick replies for vk + +v7.1.6 +---------- + * Throttle WA queues when we get 429 responses + +v7.1.5 +---------- + * Add Msg.failed_reason and set when msg fails due to reaching error limit + +v7.1.4 +---------- + * Remove loop detection now that mailroom does this + * Smarter organization of quick replies for viber keyboards + +v7.1.3 +---------- + * Use response_to_external_id instead of response_to_id + +v7.1.2 +---------- + * External channel handler should use headers config setting if provided + +v7.1.1 +---------- + * Pin to go 1.17.2 + +v7.1.0 +---------- + * Remove chatbase support + * Test with Redis 3.2.4 + * Add support for 'Expired' status in the AT handler + v7.0.0 ---------- * Tweak README diff --git a/backend.go b/backend.go index 196d02d4f..9516a0cdc 100644 --- a/backend.go +++ b/backend.go @@ -74,10 +74,6 @@ type Backend interface { // a message is being forced in being resent by a user ClearMsgSent(context.Context, MsgID) error - // IsMsgLoop returns whether the passed in message is part of a message loop, possibly with another bot. Backends should - // implement their own logic to implement this. - IsMsgLoop(ctx context.Context, msg Msg) (bool, error) - // MarkOutgoingMsgComplete marks the passed in message as having been processed. Note this should be called even in the case // of errors during sending as it will manage the number of active workers per channel. The optional status parameter can be // used to determine any sort of deduping of msg sends diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 3125bf374..a36e87888 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -12,14 +12,13 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" "github.com/nyaruka/courier/batch" - "github.com/nyaruka/courier/chatbase" "github.com/nyaruka/courier/queue" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/librato" @@ -33,17 +32,9 @@ const msgQueueName = "msgs" // the name of our set for tracking sends const sentSetName = "msgs_sent_%s" -// constants used in org configs for chatbase -const chatbaseAPIKey = "CHATBASE_API_KEY" -const chatbaseVersion = "CHATBASE_VERSION" -const chatbaseMessageType = "agent" - // our timeout for backend operations const backendTimeout = time.Second * 20 -// number of messages for loop detection -const msgLoopThreshold = 20 - func init() { courier.RegisterBackend("rapidpro", newBackend) } @@ -208,63 +199,6 @@ func (b *backend) ClearMsgSent(ctx context.Context, id courier.MsgID) error { return err } -var luaMsgLoop = redis.NewScript(3, `-- KEYS: [key, contact_id, text] - local key = KEYS[1] - local contact_id = KEYS[2] - local text = KEYS[3] - local count = 1 - - -- try to look up in window - local record = redis.call("hget", key, contact_id) - if record then - local record_count = tonumber(string.sub(record, 1, 2)) - local record_text = string.sub(record, 4, -1) - - if record_text == text then - count = math.min(record_count + 1, 99) - else - count = 1 - end - end - - -- create our new record with our updated count - record = string.format("%02d:%s", count, text) - - -- write our new record with updated count - redis.call("hset", key, contact_id, record) - - -- sets its expiration - redis.call("expire", key, 300) - - return count -`) - -// IsMsgLoop checks whether the passed in message is part of a loop -func (b *backend) IsMsgLoop(ctx context.Context, msg courier.Msg) (bool, error) { - m := msg.(*DBMsg) - - // things that aren't replies can't be loops, neither do we count retries - if m.ResponseToID_ == courier.NilMsgID || m.ErrorCount_ > 0 { - return false, nil - } - - // otherwise run our script to check whether this is a loop in the past 5 minutes - rc := b.redisPool.Get() - defer rc.Close() - - keyTime := time.Now().UTC().Round(time.Minute * 5) - key := fmt.Sprintf(sentSetName, fmt.Sprintf("loop_msgs:%s", keyTime.Format("2006-01-02-15:04"))) - count, err := redis.Int(luaMsgLoop.Do(rc, key, m.ContactID_, m.Text_)) - if err != nil { - return false, errors.Wrapf(err, "error while checking for msg loop") - } - - if count >= msgLoopThreshold { - return true, nil - } - return false, nil -} - // MarkOutgoingMsgComplete marks the passed in message as having completed processing, freeing up a worker for that channel func (b *backend) MarkOutgoingMsgComplete(ctx context.Context, msg courier.Msg, status courier.MsgStatus) { rc := b.redisPool.Get() @@ -292,16 +226,6 @@ func (b *backend) MarkOutgoingMsgComplete(ctx context.Context, msg courier.Msg, } } } - - // if this org has chatbase connected, notify chatbase - chatKey, _ := msg.Channel().OrgConfigForKey(chatbaseAPIKey, "").(string) - if chatKey != "" { - chatVersion, _ := msg.Channel().OrgConfigForKey(chatbaseVersion, "").(string) - err := chatbase.SendChatbaseMessage(chatKey, chatVersion, chatbaseMessageType, dbMsg.ContactID_.String(), msg.Channel().Name(), msg.Text(), time.Now().UTC()) - if err != nil { - logrus.WithError(err).WithField("chatbase_api_key", chatKey).WithField("chatbase_version", chatVersion).WithField("msg_id", dbMsg.ID().String()).Error("unable to write chatbase message") - } - } } // WriteMsg writes the passed in message to our store @@ -513,10 +437,39 @@ func (b *backend) Heartbeat() error { bulkSize += count } - // log our total + // get our DB and redis stats + dbStats := b.db.Stats() + redisStats := b.redisPool.Stats() + + dbWaitDurationInPeriod := dbStats.WaitDuration - b.dbWaitDuration + dbWaitCountInPeriod := dbStats.WaitCount - b.dbWaitCount + redisWaitDurationInPeriod := redisStats.WaitDuration - b.redisWaitDuration + redisWaitCountInPeriod := redisStats.WaitCount - b.redisWaitCount + + b.dbWaitDuration = dbStats.WaitDuration + b.dbWaitCount = dbStats.WaitCount + b.redisWaitDuration = redisStats.WaitDuration + b.redisWaitCount = redisStats.WaitCount + + librato.Gauge("courier.db_busy", float64(dbStats.InUse)) + librato.Gauge("courier.db_idle", float64(dbStats.Idle)) + librato.Gauge("courier.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond)) + librato.Gauge("courier.db_wait_count", float64(dbWaitCountInPeriod)) + librato.Gauge("courier.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond)) + librato.Gauge("courier.redis_wait_count", float64(redisWaitCountInPeriod)) librato.Gauge("courier.bulk_queue", float64(bulkSize)) librato.Gauge("courier.priority_queue", float64(prioritySize)) - logrus.WithField("bulk_queue", bulkSize).WithField("priority_queue", prioritySize).Info("heartbeat queue sizes calculated") + + logrus.WithFields(logrus.Fields{ + "db_busy": dbStats.InUse, + "db_idle": dbStats.Idle, + "db_wait_time": dbWaitDurationInPeriod, + "db_wait_count": dbWaitCountInPeriod, + "redis_wait_time": dbWaitDurationInPeriod, + "redis_wait_count": dbWaitCountInPeriod, + "priority_size": prioritySize, + "bulk_size": bulkSize, + }).Info("current analytics") return nil } @@ -639,11 +592,11 @@ func (b *backend) Start() error { // create our pool redisPool := &redis.Pool{ Wait: true, // makes callers wait for a connection - MaxActive: 8, // only open this many concurrent connections at once + MaxActive: 36, // only open this many concurrent connections at once MaxIdle: 4, // only keep up to this many idle IdleTimeout: 240 * time.Second, // how long to wait before reaping a connection Dial: func() (redis.Conn, error) { - conn, err := redis.Dial("tcp", fmt.Sprintf("%s", redisURL.Host)) + conn, err := redis.Dial("tcp", redisURL.Host) if err != nil { return nil, err } @@ -727,7 +680,15 @@ func (b *backend) Start() error { // create our status committer and start it b.statusCommitter = batch.NewCommitter("status committer", b.db, bulkUpdateMsgStatusSQL, time.Millisecond*500, b.committerWG, func(err error, value batch.Value) { - logrus.WithField("comp", "status committer").WithError(err).Error("error writing status") + log := logrus.WithField("comp", "status committer") + + if qerr := dbutil.AsQueryError(err); qerr != nil { + query, params := qerr.Query() + log = log.WithFields(logrus.Fields{"sql": query, "sql_params": params}) + } + + log.WithError(err).Error("error writing status") + err = courier.WriteToSpool(b.config.SpoolDir, "statuses", value) if err != nil { logrus.WithField("comp", "status committer").WithError(err).Error("error writing status to spool") @@ -738,7 +699,14 @@ func (b *backend) Start() error { // create our log committer and start it b.logCommitter = batch.NewCommitter("log committer", b.db, insertLogSQL, time.Millisecond*500, b.committerWG, func(err error, value batch.Value) { - logrus.WithField("comp", "log committer").WithError(err).Error("error writing channel log") + log := logrus.WithField("comp", "log committer") + + if qerr := dbutil.AsQueryError(err); qerr != nil { + query, params := qerr.Query() + log = log.WithFields(logrus.Fields{"sql": query, "sql_params": params}) + } + + log.WithError(err).Error("error writing channel log") }) b.logCommitter.Start() @@ -813,10 +781,13 @@ type backend struct { db *sqlx.DB redisPool *redis.Pool storage storage.Storage - awsCreds *credentials.Credentials - - popScript *redis.Script stopChan chan bool waitGroup *sync.WaitGroup + + // both sqlx and redis provide wait stats which are cummulative that we need to convert into increments + dbWaitDuration time.Duration + dbWaitCount int64 + redisWaitDuration time.Duration + redisWaitCount int64 } diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 14982c8b1..07298585e 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -17,6 +17,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/queue" + "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/null" @@ -120,7 +121,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { "sent_on": null, "high_priority": true, "channel_id": 11, - "response_to_id": 15, "response_to_external_id": "external-id", "external_id": null, "is_resend": true, @@ -137,7 +137,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.Equal(msg.ExternalID(), "") ts.Equal([]string{"Yes", "No"}, msg.QuickReplies()) ts.Equal("event", msg.Topic()) - ts.Equal(courier.NewMsgID(15), msg.ResponseToID()) ts.Equal("external-id", msg.ResponseToExternalID()) ts.True(msg.HighPriority()) ts.True(msg.IsResend()) @@ -162,8 +161,7 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { "sent_on": null, "high_priority": true, "channel_id": 11, - "response_to_id": null, - "response_to_external_id": "", + "response_to_external_id": null, "external_id": null, "metadata": null }` @@ -173,7 +171,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.NoError(err) ts.Equal([]string{}, msg.QuickReplies()) ts.Equal("", msg.Topic()) - ts.Equal(courier.NilMsgID, msg.ResponseToID()) ts.Equal("", msg.ResponseToExternalID()) ts.False(msg.IsResend()) } @@ -485,6 +482,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { ts.Equal(null.String("ext0"), m.ExternalID_) ts.True(m.ModifiedOn_.After(now)) ts.True(m.SentOn_.After(now)) + ts.Equal(null.NullString, m.FailedReason_) sentOn := *m.SentOn_ @@ -588,6 +586,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { ts.Equal(m.ErrorCount_, 1) ts.True(m.ModifiedOn_.After(now)) ts.True(m.NextAttempt_.After(now)) + ts.Equal(null.NullString, m.FailedReason_) // second go status = ts.b.NewMsgStatusForExternalID(channel, "ext1", courier.MsgErrored) @@ -598,6 +597,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { m = readMsgFromDB(ts.b, courier.NewMsgID(10000)) ts.Equal(m.Status_, courier.MsgErrored) ts.Equal(m.ErrorCount_, 2) + ts.Equal(null.NullString, m.FailedReason_) // third go status = ts.b.NewMsgStatusForExternalID(channel, "ext1", courier.MsgErrored) @@ -608,6 +608,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { m = readMsgFromDB(ts.b, courier.NewMsgID(10000)) ts.Equal(m.Status_, courier.MsgFailed) ts.Equal(m.ErrorCount_, 3) + ts.Equal(null.String("E"), m.FailedReason_) // update URN when the new doesn't exist tx, _ := ts.b.db.BeginTxx(ctx, nil) @@ -679,6 +680,11 @@ func (ts *BackendTestSuite) TestHealth() { ts.Equal(ts.b.Health(), "") } +func (ts *BackendTestSuite) TestHeartbeat() { + // TODO make analytics abstraction layer so we can test what we report + ts.NoError(ts.b.Heartbeat()) +} + func (ts *BackendTestSuite) TestDupes() { r := ts.b.redisPool.Get() defer r.Close() @@ -752,36 +758,6 @@ func (ts *BackendTestSuite) TestExternalIDDupes() { ts.True(m2.alreadyWritten) } -func (ts *BackendTestSuite) TestLoop() { - ctx := context.Background() - dbMsg := readMsgFromDB(ts.b, courier.NewMsgID(10000)) - - dbMsg.ResponseToID_ = courier.MsgID(5) - - loop, err := ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.False(loop) - - // call it 18 times more, no loop still - for i := 0; i < 18; i++ { - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.False(loop) - } - - // last one should make us a loop - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.True(loop) - - // make sure this keeps working even in hundreds of loops - for i := 0; i < 100; i++ { - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.True(loop) - } -} - func (ts *BackendTestSuite) TestStatus() { // our health should just contain the header ts.True(strings.Contains(ts.b.Status(), "Channel"), ts.b.Status()) @@ -1202,9 +1178,7 @@ func (ts *BackendTestSuite) TestSessionTimeout() { ts.NoError(err) // make sure that took - count := 0 - ts.b.db.Get(&count, "SELECT count(*) from flows_flowsession WHERE timeout_on > NOW()") - ts.Equal(1, count) + assertdb.Query(ts.T(), ts.b.db, `SELECT count(*) from flows_flowsession WHERE timeout_on > NOW()`).Returns(1) } func (ts *BackendTestSuite) TestMailroomEvents() { diff --git a/backends/rapidpro/contact.go b/backends/rapidpro/contact.go index cfec280c5..538be9421 100644 --- a/backends/rapidpro/contact.go +++ b/backends/rapidpro/contact.go @@ -9,13 +9,14 @@ import ( "unicode/utf8" "github.com/nyaruka/courier" + "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/librato" "github.com/nyaruka/null" + "github.com/pkg/errors" "github.com/jmoiron/sqlx" - "github.com/lib/pq" "github.com/sirupsen/logrus" ) @@ -103,7 +104,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne err := b.db.GetContext(ctx, contact, lookupContactFromURNSQL, urn.Identity(), org) if err != nil && err != sql.ErrNoRows { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") - return nil, err + return nil, errors.Wrap(err, "error looking up contact by URN") } // we found it, return it @@ -112,14 +113,14 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne tx, err := b.db.BeginTxx(ctx, nil) if err != nil { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") - return nil, err + return nil, errors.Wrap(err, "error beginning transaction") } err = setDefaultURN(tx, channel, contact, urn, auth) if err != nil { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") tx.Rollback() - return nil, err + return nil, errors.Wrap(err, "error setting default URN for contact") } return contact, tx.Commit() } @@ -167,13 +168,13 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne // insert it tx, err := b.db.BeginTxx(ctx, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error beginning transaction") } err = insertContact(tx, contact) if err != nil { tx.Rollback() - return nil, err + return nil, errors.Wrap(err, "error inserting contact") } // used for unit testing contact races @@ -187,13 +188,12 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne contactURN, err := contactURNForURN(tx, channel, contact.ID_, urn, auth) if err != nil { tx.Rollback() - if pqErr, ok := err.(*pq.Error); ok { + + if dbutil.IsUniqueViolation(err) { // if this was a duplicate URN, start over with a contact lookup - if pqErr.Code.Name() == "unique_violation" { - return contactForURN(ctx, b, org, channel, urn, auth, name) - } + return contactForURN(ctx, b, org, channel, urn, auth, name) } - return nil, err + return nil, errors.Wrap(err, "error getting URN for contact") } // we stole the URN from another contact, roll back and start over @@ -205,7 +205,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne // all is well, we created the new contact, commit and move forward err = tx.Commit() if err != nil { - return nil, err + return nil, errors.Wrap(err, "error commiting transaction") } // store this URN on our contact diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 6f7ef5dfd..cd96c5b4e 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -15,6 +15,7 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/pkg/errors" "mime" @@ -134,7 +135,7 @@ func writeMsgToDB(ctx context.Context, b *backend, m *DBMsg) error { // our db is down, write to the spool, we will write/queue this later if err != nil { - return err + return errors.Wrap(err, "error getting contact for message") } // set our contact and urn ids from our contact @@ -143,14 +144,14 @@ func writeMsgToDB(ctx context.Context, b *backend, m *DBMsg) error { rows, err := b.db.NamedQueryContext(ctx, insertMsgSQL, m) if err != nil { - return err + return errors.Wrap(err, "error inserting message") } defer rows.Close() rows.Next() err = rows.Scan(&m.ID_) if err != nil { - return err + return errors.Wrap(err, "error scanning for inserted message id") } // queue this up to be handled by RapidPro @@ -175,6 +176,7 @@ SELECT attachments, msg_count, error_count, + failed_reason, high_priority, status, visibility, @@ -489,7 +491,6 @@ type DBMsg struct { Text_ string `json:"text" db:"text"` Attachments_ pq.StringArray `json:"attachments" db:"attachments"` ExternalID_ null.String `json:"external_id" db:"external_id"` - ResponseToID_ courier.MsgID `json:"response_to_id" db:"response_to_id"` ResponseToExternalID_ string `json:"response_to_external_id"` IsResend_ bool `json:"is_resend,omitempty"` Metadata_ json.RawMessage `json:"metadata" db:"metadata"` @@ -498,8 +499,9 @@ type DBMsg struct { ContactID_ ContactID `json:"contact_id" db:"contact_id"` ContactURNID_ ContactURNID `json:"contact_urn_id" db:"contact_urn_id"` - MessageCount_ int `json:"msg_count" db:"msg_count"` - ErrorCount_ int `json:"error_count" db:"error_count"` + MessageCount_ int `json:"msg_count" db:"msg_count"` + ErrorCount_ int `json:"error_count" db:"error_count"` + FailedReason_ null.String `json:"failed_reason" db:"failed_reason"` ChannelUUID_ courier.ChannelUUID `json:"channel_uuid"` ContactName_ string `json:"contact_name"` @@ -534,7 +536,6 @@ func (m *DBMsg) ContactName() string { return m.ContactName_ } func (m *DBMsg) HighPriority() bool { return m.HighPriority_ } func (m *DBMsg) ReceivedOn() *time.Time { return m.SentOn_ } func (m *DBMsg) SentOn() *time.Time { return m.SentOn_ } -func (m *DBMsg) ResponseToID() courier.MsgID { return m.ResponseToID_ } func (m *DBMsg) ResponseToExternalID() string { return m.ResponseToExternalID_ } func (m *DBMsg) IsResend() bool { return m.IsResend_ } diff --git a/backends/rapidpro/schema.sql b/backends/rapidpro/schema.sql index 191169480..e21cbb7c0 100644 --- a/backends/rapidpro/schema.sql +++ b/backends/rapidpro/schema.sql @@ -72,6 +72,7 @@ CREATE TABLE msgs_msg ( msg_count integer NOT NULL, error_count integer NOT NULL, next_attempt timestamp with time zone NOT NULL, + failed_reason character varying(1), external_id character varying(255), attachments character varying(255)[], channel_id integer references channels_channel(id) on delete cascade, @@ -79,7 +80,8 @@ CREATE TABLE msgs_msg ( contact_urn_id integer NOT NULL references contacts_contacturn(id) on delete cascade, org_id integer NOT NULL references orgs_org(id) on delete cascade, metadata text, - topup_id integer + topup_id integer, + delete_from_counts boolean ); DROP TABLE IF EXISTS channels_channellog CASCADE; diff --git a/backends/rapidpro/status.go b/backends/rapidpro/status.go index 6a81d6335..56759fcf2 100644 --- a/backends/rapidpro/status.go +++ b/backends/rapidpro/status.go @@ -107,6 +107,14 @@ UPDATE msgs_msg SET ELSE next_attempt END, + failed_reason = CASE + WHEN + error_count >= 2 + THEN + 'E' + ELSE + failed_reason + END, sent_on = CASE WHEN :status = 'W' @@ -164,6 +172,14 @@ UPDATE msgs_msg SET ELSE next_attempt END, + failed_reason = CASE + WHEN + error_count >= 2 + THEN + 'E' + ELSE + failed_reason + END, sent_on = CASE WHEN :status IN ('W', 'S', 'D') @@ -288,8 +304,6 @@ WHERE msgs_msg.id = s.msg_id::bigint AND msgs_msg.channel_id = s.channel_id::int AND msgs_msg.direction = 'O' -RETURNING - msgs_msg.id ` //----------------------------------------------------------------------------- diff --git a/backends/rapidpro/urn.go b/backends/rapidpro/urn.go index 4e25cddd1..6e71e9402 100644 --- a/backends/rapidpro/urn.go +++ b/backends/rapidpro/urn.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/nyaruka/null" + "github.com/pkg/errors" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" @@ -209,14 +210,14 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn } err := db.Get(contactURN, selectOrgURN, channel.OrgID(), urn.Identity()) if err != nil && err != sql.ErrNoRows { - return nil, err + return nil, errors.Wrap(err, "error looking up URN by identity") } // we didn't find it, let's insert it if err == sql.ErrNoRows { err = insertContactURN(db, contactURN) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error inserting URN") } } @@ -232,7 +233,7 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn contactURN.Display = display err = updateContactURN(db, contactURN) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error updating URN") } } @@ -242,7 +243,7 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn err = updateContactURN(db, contactURN) } - return contactURN, err + return contactURN, errors.Wrap(err, "error updating URN auth") } const insertURN = ` diff --git a/batch/batch.go b/batch/batch.go index bfec78c46..1848ce911 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -2,11 +2,11 @@ package batch import ( "context" - "strings" "sync" "time" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/dbutil" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -147,7 +147,7 @@ func (c *committer) flush(size int) bool { err = batchSQL(ctx, c.label, c.db, c.sql, []interface{}{v}) if err != nil { if c.callback != nil { - c.callback(errors.Wrapf(err, "%s: error comitting value", c.label), v.(Value)) + c.callback(errors.Wrapf(err, "%s: error committing value", c.label), v.(Value)) } } } @@ -180,86 +180,12 @@ func batchSQL(ctx context.Context, label string, db *sqlx.DB, sql string, vs []i start := time.Now() - // this will be our SQL placeholders ($1, $2,..) for values in our final query, built dynamically - values := strings.Builder{} - values.Grow(7 * len(vs)) - - // this will be each of the arguments to match the positional values above - args := make([]interface{}, 0, len(vs)*5) - - // for each value we build a bound SQL statement, then extract the values clause - for i, value := range vs { - valueSQL, valueArgs, err := sqlx.Named(sql, value) - if err != nil { - return errors.Wrapf(err, "error converting bulk insert args") - } - - args = append(args, valueArgs...) - argValues, err := extractValues(valueSQL) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", valueSQL) - } - - // append to our global values, adding comma if necessary - values.WriteString(argValues) - if i+1 < len(vs) { - values.WriteString(",") - } - } - - valuesSQL, err := extractValues(sql) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", sql) - } - - bulkInsert := db.Rebind(strings.Replace(sql, valuesSQL, values.String(), -1)) - - // insert them all at once - rows, err := db.QueryxContext(ctx, bulkInsert, args...) + err := dbutil.BulkQuery(ctx, db, sql, vs) if err != nil { - return errors.Wrapf(err, "error during bulk insert") - } - defer rows.Close() - - // iterate our remaining rows - for rows.Next() { - } - - // check for any error - if rows.Err() != nil { - return errors.Wrapf(rows.Err(), "error in row cursor") + return err } logrus.WithField("elapsed", time.Since(start)).WithField("rows", len(vs)).Infof("%s bulk sql complete", label) return nil } - -// extractValues extracts the portion between `VALUE(` and `)` in the passed in string. (leaving VALUE but not the parentheses) -func extractValues(sql string) (string, error) { - startValues := strings.Index(sql, "VALUES(") - if startValues <= 0 { - return "", errors.Errorf("unable to find VALUES( in bulk insert SQL: %s", sql) - } - - // find the matching end parentheses, we need to count balanced parentheses here - openCount := 1 - endValues := -1 - for i, r := range sql[startValues+7:] { - if r == '(' { - openCount++ - } else if r == ')' { - openCount-- - if openCount == 0 { - endValues = i + startValues + 7 - break - } - } - } - - if endValues <= 0 { - return "", errors.Errorf("unable to find end of VALUES() in bulk insert sql: %s", sql) - } - - return sql[startValues+6 : endValues+1], nil -} diff --git a/batch/batch_test.go b/batch/batch_test.go index 73947201d..0c2de2d2d 100644 --- a/batch/batch_test.go +++ b/batch/batch_test.go @@ -8,6 +8,7 @@ import ( "github.com/jmoiron/sqlx" _ "github.com/lib/pq" + "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/stretchr/testify/assert" ) @@ -44,9 +45,7 @@ func TestBatchInsert(t *testing.T) { time.Sleep(time.Second) assert.NoError(t, callbackErr) - count := 0 - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 3, count) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(3) committer.Queue(&Label{0, "label4"}) committer.Queue(&Label{0, "label3"}) @@ -54,9 +53,8 @@ func TestBatchInsert(t *testing.T) { time.Sleep(time.Second) assert.Error(t, callbackErr) - assert.Equal(t, `labels: error comitting value: error during bulk insert: pq: duplicate key value violates unique constraint "labels_label_key"`, callbackErr.Error()) - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 4, count) + assert.Equal(t, `labels: error committing value: error making bulk query: pq: duplicate key value violates unique constraint "labels_label_key"`, callbackErr.Error()) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(4) } func TestBatchUpdate(t *testing.T) { @@ -94,17 +92,8 @@ func TestBatchUpdate(t *testing.T) { time.Sleep(time.Second) assert.NoError(t, callbackErr) - count := 0 - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 3, count) - - label := "" - db.Get(&label, "SELECT label FROM labels WHERE id = 1;") - assert.Equal(t, "label001", label) - - db.Get(&label, "SELECT label FROM labels WHERE id = 2;") - assert.Equal(t, "label02", label) - - db.Get(&label, "SELECT label FROM labels WHERE id = 3;") - assert.Equal(t, "label03", label) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(3) + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 1`).Returns("label001") + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 2`).Returns("label02") + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 3`).Returns("label03") } diff --git a/channel.go b/channel.go index d64394cca..7a71a40d1 100644 --- a/channel.go +++ b/channel.go @@ -52,6 +52,9 @@ const ( // ConfigUseNational is a constant key for channel configs ConfigUseNational = "use_national" + + // ConfigSendHeaders is a constant key for channel configs + ConfigSendHeaders = "headers" ) // ChannelType is our typing of the two char channel types diff --git a/chatbase/chatbase.go b/chatbase/chatbase.go deleted file mode 100644 index e5bca39c9..000000000 --- a/chatbase/chatbase.go +++ /dev/null @@ -1,50 +0,0 @@ -package chatbase - -import ( - "bytes" - "encoding/json" - "net/http" - "time" - - "github.com/nyaruka/courier/utils" -) - -// ChatbaseAPIURL is the URL chatbase API messages will be sent to -var chatbaseAPIURL = "https://chatbase.com/api/message" - -// chatbaseLog is the payload for a chatbase request -type chatbaseLog struct { - Type string `json:"type"` - UserID string `json:"user_id"` - Platform string `json:"platform"` - Message string `json:"message"` - TimeStamp int64 `json:"time_stamp"` - - APIKey string `json:"api_key"` - APIVersion string `json:"version,omitempty"` -} - -// SendChatbaseMessage sends a chatbase message with the passed in api key and message details -func SendChatbaseMessage(apiKey string, apiVersion string, messageType string, userID string, platform string, message string, timestamp time.Time) error { - body := chatbaseLog{ - Type: messageType, - UserID: userID, - Platform: platform, - Message: message, - TimeStamp: timestamp.UnixNano() / int64(time.Millisecond), - - APIKey: apiKey, - APIVersion: apiVersion, - } - - jsonBody, err := json.Marshal(body) - if err != nil { - return err - } - - req, _ := http.NewRequest(http.MethodPost, chatbaseAPIURL, bytes.NewReader(jsonBody)) - req.Header.Set("Content-Type", "application/json") - - _, err = utils.MakeHTTPRequest(req) - return err -} diff --git a/chatbase/chatbase_test.go b/chatbase/chatbase_test.go deleted file mode 100644 index 12ee9dbe7..000000000 --- a/chatbase/chatbase_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package chatbase - -import ( - "bytes" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/buger/jsonparser" - "github.com/stretchr/testify/assert" -) - -func TestChatbase(t *testing.T) { - var testRequest *http.Request - var statusCode = 200 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := ioutil.ReadAll(r.Body) - testRequest = httptest.NewRequest(r.Method, r.URL.String(), bytes.NewBuffer(body)) - testRequest.Header = r.Header - w.WriteHeader(statusCode) - w.Write([]byte("ok")) - })) - defer server.Close() - - chatbaseAPIURL = server.URL - - now := time.Now() - err := SendChatbaseMessage("apiKey", "apiVersion", "messageType", "userID", "platform", "message", now) - assert.NoError(t, err) - - // parse our body - bytes, err := ioutil.ReadAll(testRequest.Body) - assert.NoError(t, err) - - // check our request body - str, err := jsonparser.GetString(bytes, "type") - assert.NoError(t, err) - assert.Equal(t, "messageType", str) - - str, err = jsonparser.GetString(bytes, "version") - assert.NoError(t, err) - assert.Equal(t, "apiVersion", str) - - ts, err := jsonparser.GetInt(bytes, "time_stamp") - assert.NoError(t, err) - assert.Equal(t, now.UnixNano()/int64(time.Millisecond), ts) - - // simulate an error - statusCode = 500 - err = SendChatbaseMessage("apiKey", "apiVersion", "messageType", "userID", "platform", "message", now) - assert.Error(t, err) - - // simulate error when messageType is invalid - statusCode = 400 - err = SendChatbaseMessage("apiKey", "apiVersion", "msg", "userID", "platform", "message", now) - assert.Error(t, err) - - bytes, err = ioutil.ReadAll(testRequest.Body) - str, err = jsonparser.GetString(bytes, "type") - assert.NoError(t, err) - assert.Equal(t, "msg", str) -} diff --git a/go.mod b/go.mod index ccb11b67c..bb6f37861 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/nyaruka/courier +go 1.17 + require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect @@ -11,29 +13,27 @@ require ( github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 - github.com/go-playground/locales v0.11.2 // indirect - github.com/go-playground/universal-translator v0.16.0 // indirect - github.com/go-sql-driver/mysql v1.5.0 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible - github.com/gomodule/redigo v2.0.0+incompatible + github.com/gomodule/redigo v1.8.8 github.com/gorilla/schema v1.0.2 - github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 + github.com/jmoiron/sqlx v1.3.4 github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect - github.com/lib/pq v1.0.0 - github.com/mattn/go-sqlite3 v1.10.0 // indirect + github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.14.1 + github.com/nyaruka/gocommon v1.17.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 + github.com/nyaruka/redisx v0.2.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 github.com/stretchr/testify v1.7.0 golang.org/x/mod v0.4.2 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/go-playground/assert.v1 v1.2.1 // indirect - gopkg.in/go-playground/validator.v9 v9.11.0 + gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/h2non/filetype.v1 v1.0.5 ) @@ -43,6 +43,7 @@ require ( github.com/golang/protobuf v1.3.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect github.com/nyaruka/phonenumbers v1.0.71 // indirect @@ -53,5 +54,3 @@ require ( golang.org/x/text v0.3.6 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) - -go 1.17 diff --git a/go.sum b/go.sum index 606fbc4d4..b8190da2f 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TR github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/aws/aws-sdk-go v1.34.31/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 h1:SnUWpAH4lEUoS86woR12h21VMUbDe+DYp88V646wwMI= @@ -23,16 +24,18 @@ github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyN github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-playground/locales v0.11.2 h1:wH6Ksuvzk0SU9M6wUeGz/EaRWnavAHCOsFre1njzgi8= -github.com/go-playground/locales v0.11.2/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= +github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= @@ -41,8 +44,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -52,24 +55,38 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= +github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= github.com/nyaruka/gocommon v1.14.1 h1:/ScvLmg4zzVAuZ78TaENrvSEvW3WnUdqRd/t9hX7z7E= github.com/nyaruka/gocommon v1.14.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.15.0 h1:n0jdOEkorVIxyD+53XHRi49d/C3hD1q70o9HILnfz9U= +github.com/nyaruka/gocommon v1.15.0/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.16.0 h1:F2DXo8075ErYm3pIJ5209HXRIyGWvwH8mtyxjwRYN0w= +github.com/nyaruka/gocommon v1.16.0/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= +github.com/nyaruka/gocommon v1.17.0 h1:cTiDLSUgmYJ9OZw752jva0P2rz0utRtv5WGuKFc9kxw= +github.com/nyaruka/gocommon v1.17.0/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= github.com/nyaruka/null v1.1.1/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= +github.com/nyaruka/phonenumbers v1.0.58/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/nyaruka/phonenumbers v1.0.71 h1:itkCGhxkQkHrJ6OyZSApdjQVlPmrWs88MF283pPvbFU= github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/redisx v0.2.1 h1:BavpQRCsK5xV2uxPdJJ26yVmjSo+q6bdjWqeNNf0s5w= +github.com/nyaruka/redisx v0.2.1/go.mod h1:cdbAm4y/+oFWu7qFzH2ERPeqRXJC2CtgRhwcBacM4Oc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -85,25 +102,31 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -115,8 +138,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.11.0 h1:ER548TqE6ZknRRDDo0/tP8I12UHYxNlIfss8tMd4iCo= -gopkg.in/go-playground/validator.v9 v9.11.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handlers/africastalking/africastalking.go b/handlers/africastalking/africastalking.go index b64553d59..e196212fb 100644 --- a/handlers/africastalking/africastalking.go +++ b/handlers/africastalking/africastalking.go @@ -91,6 +91,7 @@ var statusMapping = map[string]courier.MsgStatusValue{ "Buffered": courier.MsgSent, "Rejected": courier.MsgFailed, "Failed": courier.MsgFailed, + "Expired": courier.MsgFailed, } // receiveStatus is our HTTP handler function for status updates @@ -105,7 +106,7 @@ func (h *handler) receiveStatus(ctx context.Context, channel courier.Channel, w msgStatus, found := statusMapping[form.Status] if !found { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, - fmt.Errorf("unknown status '%s', must be one of 'Success','Sent','Buffered','Rejected' or 'Failed'", form.Status)) + fmt.Errorf("unknown status '%s', must be one of 'Success','Sent','Buffered','Rejected', 'Failed', or 'Expired'", form.Status)) } // write our status diff --git a/handlers/africastalking/africastalking_test.go b/handlers/africastalking/africastalking_test.go index cc36711e3..c3941a58a 100644 --- a/handlers/africastalking/africastalking_test.go +++ b/handlers/africastalking/africastalking_test.go @@ -26,7 +26,8 @@ var ( missingStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7" invalidStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Borked" - validStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Success" + successStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Success" + expiredStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Expired" ) var testCases = []ChannelHandleTestCase{ @@ -42,7 +43,8 @@ var testCases = []ChannelHandleTestCase{ {Label: "Invalid Date", URL: receiveURL, Data: invalidDate, Status: 400, Response: "invalid date format"}, {Label: "Status Invalid", URL: statusURL, Status: 400, Data: invalidStatus, Response: "unknown status"}, {Label: "Status Missing", URL: statusURL, Status: 400, Data: missingStatus, Response: "field 'status' required"}, - {Label: "Status Valid", URL: statusURL, Status: 200, Data: validStatus, Response: `"status":"D"`}, + {Label: "Status Success", URL: statusURL, Status: 200, Data: successStatus, Response: `"status":"D"`}, + {Label: "Status Expired", URL: statusURL, Status: 200, Data: expiredStatus, Response: `"status":"F"`}, } func TestHandler(t *testing.T) { diff --git a/handlers/chikka/chikka_test.go b/handlers/chikka/chikka_test.go index dd6fbc38a..b31a9533e 100644 --- a/handlers/chikka/chikka_test.go +++ b/handlers/chikka/chikka_test.go @@ -71,7 +71,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Plain Reply", Text: "Simple Message", URN: "tel:+63911231234", Status: "W", - ResponseToID: 5, ResponseToExternalID: "external-id", ResponseBody: "Success", ResponseStatus: 200, PostParams: map[string]string{ @@ -88,7 +87,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Failed Reply use Send", Text: "Simple Message", URN: "tel:+63911231234", - ResponseToID: 5, ResponseToExternalID: "external-id", ResponseBody: `{"status":400,"message":"BAD REQUEST","description":"Invalid\\/Used Request ID"}`, ResponseStatus: 400, diff --git a/handlers/external/external.go b/handlers/external/external.go index c447228d2..da01ecce2 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -353,11 +353,17 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } req.Header.Set("Content-Type", contentTypeHeader) + // TODO can drop this when channels have been migrated to use ConfigSendHeaders authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") if authorization != "" { req.Header.Set("Authorization", authorization) } + headers := msg.Channel().ConfigForKey(courier.ConfigSendHeaders, map[string]interface{}{}).(map[string]interface{}) + for hKey, hValue := range headers { + req.Header.Set(hKey, fmt.Sprint(hValue)) + } + rr, err := utils.MakeHTTPRequest(req) // record our status and log diff --git a/handlers/external/external_test.go b/handlers/external/external_test.go index 1c4b84685..8a9343c68 100644 --- a/handlers/external/external_test.go +++ b/handlers/external/external_test.go @@ -423,11 +423,11 @@ func TestSending(t *testing.T) { var jsonChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) var xmlChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", @@ -472,22 +472,22 @@ func TestSending(t *testing.T) { var jsonChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) var xmlChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, - courier.ConfigContentType: contentXML, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, + courier.ConfigContentType: contentXML, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) RunChannelSendTestCases(t, getChannel30IntLength, newHandler(), longSendTestCases, nil) @@ -503,4 +503,14 @@ func TestSending(t *testing.T) { RunChannelSendTestCases(t, nationalChannel, newHandler(), nationalGetSendTestCases, nil) + var jsonChannelWithSendAuthorization = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendAuthorization: "Token ABCDEF", + }) + RunChannelSendTestCases(t, jsonChannelWithSendAuthorization, newHandler(), jsonSendTestCases, nil) + } diff --git a/handlers/facebook/facebook.go b/handlers/facebook/facebook.go index 62fd688bc..75aff3de4 100644 --- a/handlers/facebook/facebook.go +++ b/handlers/facebook/facebook.go @@ -481,7 +481,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" diff --git a/handlers/facebook/facebook_test.go b/handlers/facebook/facebook_test.go index 6a22ebbc9..42f2ad648 100644 --- a/handlers/facebook/facebook_test.go +++ b/handlers/facebook/facebook_test.go @@ -569,7 +569,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 92bb45e83..6bf94f712 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -23,13 +23,13 @@ import ( // Endpoints we hit var ( - sendURL = "https://graph.facebook.com/v7.0/me/messages" - graphURL = "https://graph.facebook.com/v7.0/" + sendURL = "https://graph.facebook.com/v12.0/me/messages" + graphURL = "https://graph.facebook.com/v12.0/" signatureHeader = "X-Hub-Signature" - // Facebook API says 640 is max for the body - maxMsgLength = 640 + // max for the body + maxMsgLength = 1000 // Sticker ID substitutions stickerIDToEmoji = map[int64]string{ @@ -56,18 +56,20 @@ const ( payloadKey = "payload" ) +func newHandler(channelType courier.ChannelType, name string, useUUIDRoutes bool) courier.ChannelHandler { + return &handler{handlers.NewBaseHandlerWithParams(channelType, name, useUUIDRoutes)} +} + func init() { - courier.RegisterHandler(newHandler()) + courier.RegisterHandler(newHandler("IG", "Instagram", false)) + courier.RegisterHandler(newHandler("FBA", "Facebook", false)) + } type handler struct { handlers.BaseHandler } -func newHandler() courier.ChannelHandler { - return &handler{handlers.NewBaseHandlerWithParams(courier.ChannelType("FBA"), "Facebook", false)} -} - // Initialize is called by the engine once everything is loaded func (h *handler) Initialize(s courier.Server) error { h.SetServer(s) @@ -76,12 +78,12 @@ func (h *handler) Initialize(s courier.Server) error { return nil } -type fbSender struct { +type Sender struct { ID string `json:"id"` - UserRef string `json:"user_ref"` + UserRef string `json:"user_ref,omitempty"` } -type fbUser struct { +type User struct { ID string `json:"id"` } @@ -108,9 +110,9 @@ type moPayload struct { ID string `json:"id"` Time int64 `json:"time"` Messaging []struct { - Sender fbSender `json:"sender"` - Recipient fbUser `json:"recipient"` - Timestamp int64 `json:"timestamp"` + Sender Sender `json:"sender"` + Recipient User `json:"recipient"` + Timestamp int64 `json:"timestamp"` OptIn *struct { Ref string `json:"ref"` @@ -125,6 +127,7 @@ type moPayload struct { } `json:"referral"` Postback *struct { + MID string `json:"mid"` Title string `json:"title"` Payload string `json:"payload"` Referral struct { @@ -188,9 +191,9 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, err } - // not a page object? ignore - if payload.Object != "page" { - return nil, fmt.Errorf("object expected 'page', found %s", payload.Object) + // is not a 'page' and 'instagram' object? ignore it + if payload.Object != "page" && payload.Object != "instagram" { + return nil, fmt.Errorf("object expected 'page' or 'instagram', found %s", payload.Object) } // no entries? ignore this request @@ -198,9 +201,14 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, fmt.Errorf("no entries found") } - pageID := payload.Entry[0].ID + entryID := payload.Entry[0].ID - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(pageID)) + //if object is 'page' returns type FBA, if object is 'instagram' returns type IG + if payload.Object == "page" { + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(entryID)) + } else { + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(entryID)) + } } // receiveVerify handles Facebook's webhook verification callback @@ -235,9 +243,9 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } - // not a page object? ignore - if payload.Object != "page" { - return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring non-page request") + // is not a 'page' and 'instagram' object? ignore it + if payload.Object != "page" && payload.Object != "instagram" { + return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") } // no entries? ignore this request @@ -274,11 +282,21 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h sender = msg.Sender.ID } + var urn urns.URN + // create our URN - urn, err := urns.NewFacebookURN(sender) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + if payload.Object == "instagram" { + urn, err = urns.NewInstagramURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + } else { + urn, err = urns.NewFacebookURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } } + if msg.OptIn != nil { // this is an opt in, if we have a user_ref, use that as our URN (this is a checkbox plugin) // TODO: @@ -396,6 +414,11 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h attachmentURLs = append(attachmentURLs, fmt.Sprintf("geo:%f,%f", att.Payload.Coordinates.Lat, att.Payload.Coordinates.Long)) } + if att.Type == "story_mention" { + data = append(data, courier.NewInfoData("ignoring story_mention")) + continue + } + if att.Payload != nil && att.Payload.URL != "" { attachmentURLs = append(attachmentURLs, att.Payload.URL) } @@ -528,13 +551,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" payload.Tag = tagByTopic[topic] } else { - payload.MessagingType = "NON_PROMOTIONAL_SUBSCRIPTION" // only allowed until Jan 15, 2020 + payload.MessagingType = "UPDATE" } // build our recipient @@ -796,7 +819,6 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn u := base.ResolveReference(path) query := url.Values{} - query.Set("fields", "first_name,last_name") query.Set("access_token", accessToken) u.RawQuery = query.Encode() req, _ := http.NewRequest(http.MethodGet, u.String(), nil) @@ -805,11 +827,10 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) } - // read our first and last name - firstName, _ := jsonparser.GetString(rr.Body, "first_name") - lastName, _ := jsonparser.GetString(rr.Body, "last_name") + // read our name + name, _ := jsonparser.GetString(rr.Body, "name") - return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil + return map[string]string{"name": name}, nil } // see https://developers.facebook.com/docs/messenger-platform/webhook#security diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 10c727716..1a402116d 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -16,21 +16,25 @@ import ( "github.com/stretchr/testify/assert" ) -var testChannels = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "1234", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +var testChannelsFBA = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } -var helloMsg = `{ +var testChannelsIG = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +} + +var helloMsgFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -41,17 +45,76 @@ var helloMsg = `{ }] }` -var duplicateMsg = `{ +var helloMsgIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var duplicateMsgFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }, + { + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var duplicateMsgIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -61,14 +124,14 @@ var duplicateMsg = `{ "time": 1459991487970 }, { - "id": "1234", + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -79,17 +142,38 @@ var duplicateMsg = `{ }] }` -var invalidURN = `{ +var invalidURNFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "abc5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var invalidURNIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "abc5678" @@ -100,10 +184,10 @@ var invalidURN = `{ }] }` -var attachment = `{ +var attachmentFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "mid": "external_id", @@ -115,7 +199,33 @@ var attachment = `{ }] }, "recipient": { - "id": "1234" + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var attachmentIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"image", + "payload":{ + "url":"https://image-url/foo.png" + } + }] + }, + "recipient": { + "id": "12345" }, "sender": { "id": "5678" @@ -129,7 +239,7 @@ var attachment = `{ var locationAttachment = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "mid": "external_id", @@ -144,7 +254,7 @@ var locationAttachment = `{ }] }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -158,11 +268,11 @@ var locationAttachment = `{ var thumbsUp = `{ "object":"page", "entry":[{ - "id":"1234", + "id":"12345", "time":1459991487970, "messaging":[{ "sender":{"id":"5678"}, - "recipient":{"id":"1234"}, + "recipient":{"id":"12345"}, "timestamp":1459991487970, "message":{ "mid":"external_id", @@ -178,10 +288,50 @@ var thumbsUp = `{ }] }` -var differentPage = `{ +var like_heart = `{ + "object":"instagram", + "entry":[{ + "id":"12345", + "messaging":[{ + "sender":{"id":"5678"}, + "recipient":{"id":"12345"}, + "timestamp":1459991487970, + "message":{ + "mid":"external_id", + "attachments":[{ + "type":"like_heart" + }] + } + }], + "time":1459991487970 + }] +}` + +var differentPageIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1235" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var differentPageFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "text": "Hello World", @@ -199,13 +349,33 @@ var differentPage = `{ }] }` -var echo = `{ +var echoFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970, + "message": { + "is_echo": true, + "mid": "qT7ywaK" + } + }] + }] +}` + +var echoIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -219,17 +389,38 @@ var echo = `{ }] }` +var icebreakerGetStarted = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "postback": { + "title": "icebreaker question", + "payload": "get_started" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + var optInUserRef = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "optin": { "ref": "optin_ref", "user_ref": "optin_user_ref" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -243,13 +434,13 @@ var optInUserRef = `{ var optIn = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "optin": { "ref": "optin_ref" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -263,7 +454,7 @@ var optIn = `{ var postback = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", @@ -275,7 +466,7 @@ var postback = `{ } }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -289,7 +480,7 @@ var postback = `{ var postbackReferral = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", @@ -302,7 +493,7 @@ var postbackReferral = `{ } }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -316,14 +507,14 @@ var postbackReferral = `{ var postbackGetStarted = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", "payload": "get_started" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -337,7 +528,7 @@ var postbackGetStarted = `{ var referral = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "referral": { "ref": "referral id", @@ -346,7 +537,7 @@ var referral = `{ "type": "referral type" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678", @@ -361,7 +552,7 @@ var referral = `{ var dlr = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "delivery":{ "mids":[ @@ -371,7 +562,7 @@ var dlr = `{ "seq":37 }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -387,25 +578,58 @@ var notPage = `{ "entry": [{}] }` -var noEntries = `{ +var notInstagram = `{ + "object":"notinstagram", + "entry": [{}] +}` + +var noEntriesFBA = `{ "object":"page", "entry": [] }` -var noMessagingEntries = `{ +var noEntriesIG = `{ + "object":"instagram", + "entry": [] +}` + +var noMessagingEntriesFBA = `{ "object":"page", "entry": [{ - "id": "1234" + "id": "12345" }] }` -var unkownMessagingEntry = `{ +var noMessagingEntriesIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345" + }] +}` + +var unknownMessagingEntryFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }] + }] +}` + +var unknownMessagingEntryIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -419,12 +643,12 @@ var customerFeedbackResponse = `{ "object": "page", "entry": [ { - "id": "1234", + "id": "12345", "time": 1459991487970, "messaging": [ { "recipient": { - "id": "1234" + "id": "12345" }, "timestamp": 1459991487970, "sender": { @@ -449,20 +673,44 @@ var customerFeedbackResponse = `{ ] } ` +var storyMentionIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"story_mention", + "payload":{ + "url":"https://story-url" + } + }] + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` var notJSON = `blargh` -var testCases = []ChannelHandleTestCase{ - {Label: "Receive Message", URL: "/c/fba/receive", Data: helloMsg, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, +var testCasesFBA = []ChannelHandleTestCase{ + {Label: "Receive Message FBA", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, + {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: helloMsg, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - - {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: duplicateMsg, Status: 200, Response: "Handled", + {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: duplicateMsgFBA, Status: 200, Response: "Handled", Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Attachment", URL: "/c/fba/receive", Data: attachment, Status: 200, Response: "Handled", + {Label: "Receive Attachment", URL: "/c/fba/receive", Data: attachmentFBA, Status: 200, Response: "Handled", Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, @@ -504,19 +752,52 @@ var testCases = []ChannelHandleTestCase{ Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), MsgStatus: Sp(courier.MsgDelivered), ExternalID: Sp("mid.1458668856218:ed81099e15d3f4f233"), PrepRequest: addValidSignature}, - {Label: "Different Page", URL: "/c/fba/receive", Data: differentPage, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, - {Label: "Echo", URL: "/c/fba/receive", Data: echo, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "expected 'page', found notpage", PrepRequest: addValidSignature}, - {Label: "No Entries", URL: "/c/fba/receive", Data: noEntries, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntries, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unkownMessagingEntry, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Different Page", URL: "/c/fba/receive", Data: differentPageFBA, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/fba/receive", Data: echoFBA, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/fba/receive", Data: noEntriesFBA, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntriesFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unknownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/fba/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURN, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, - + {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURNFBA, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, {Label: "Receive Customer Feedback Message", URL: "/c/fba/receive", Data: customerFeedbackResponse, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("4"), URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, } +var testCasesIG = []ChannelHandleTestCase{ + {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsgIG, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsgIG, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, + + {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsgIG, Status: 200, Response: "Handled", + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachmentIG, Status: 200, Response: "Handled", + Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", + Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", + URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), + ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, + PrepRequest: addValidSignature}, + + {Label: "Different Page", URL: "/c/ig/receive", Data: differentPageIG, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/ig/receive", Data: echoIG, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/ig/receive", Data: noEntriesIG, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unknownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, + {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, +} func addValidSignature(r *http.Request) { body, _ := handlers.ReadBody(r, 100000) @@ -541,23 +822,23 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { // user has a name if strings.HasSuffix(r.URL.Path, "1337") { - w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + w.Write([]byte(`{ "name": "John Doe"}`)) return } // no name - w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + w.Write([]byte(`{ "name": ""}`)) })) graphURL = server.URL return server } -func TestDescribe(t *testing.T) { - fbGraph := buildMockFBGraph(testCases) +func TestDescribeFBA(t *testing.T) { + fbGraph := buildMockFBGraph(testCasesFBA) defer fbGraph.Close() - handler := newHandler().(courier.URNDescriber) + handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) tcs := []struct { urn urns.URN metadata map[string]string @@ -566,25 +847,49 @@ func TestDescribe(t *testing.T) { {"facebook:ref:1337", map[string]string{}}} for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) + metadata, _ := handler.DescribeURN(context.Background(), testChannelsFBA[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } +} + +func TestDescribeIG(t *testing.T) { + fbGraph := buildMockFBGraph(testCasesIG) + defer fbGraph.Close() + + handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"instagram:1337", map[string]string{"name": "John Doe"}}, + {"instagram:4567", map[string]string{"name": ""}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsIG[0], tc.urn) assert.Equal(t, metadata, tc.metadata) } } func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), testCases) + RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) + RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) + } func BenchmarkHandler(b *testing.B) { - fbService := buildMockFBGraph(testCases) - defer fbService.Close() + fbService := buildMockFBGraph(testCasesFBA) + + RunChannelBenchmarks(b, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) + fbService.Close() - RunChannelBenchmarks(b, testChannels, newHandler(), testCases) + fbServiceIG := buildMockFBGraph(testCasesIG) + + RunChannelBenchmarks(b, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) + fbServiceIG.Close() } func TestVerify(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ + RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), []ChannelHandleTestCase{ {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, {Label: "Verify No Mode", URL: "/c/fba/receive", Status: 400, Response: "unknown request"}, @@ -593,6 +898,15 @@ func TestVerify(t *testing.T) { {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, }) + RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), []ChannelHandleTestCase{ + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, + Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, + {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, + {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, + {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, + }) + } // setSendURL takes care of setting the send_url to our test server host @@ -600,16 +914,16 @@ func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, sendURL = s.URL } -var defaultSendTestCases = []ChannelSendTestCase{ +var SendTestCasesFBA = []ChannelSendTestCase{ {Label: "Plain Send", Text: "Simple Message", URN: "facebook:12345", Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, @@ -618,13 +932,13 @@ var defaultSendTestCases = []ChannelSendTestCase{ ContactURNs: map[string]bool{"facebook:12345": true, "ext:67890": true, "facebook:ref:67890": false}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133", "recipient_id": "12345"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Quick Reply", Text: "Are you happy?", URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Long Message", Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", @@ -637,7 +951,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "Send caption and photo with Quick Reply", Text: "This is some text.", @@ -651,7 +965,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "facebook:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "ID Error", Text: "ID Error", URN: "facebook:12345", @@ -665,11 +979,77 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, } +var SendTestCasesIG = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Plain Response", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Quick Reply", + Text: "Are you happy?", URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Long Message", + Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", + URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Send Photo", + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + SendPrep: setSendURL}, + {Label: "Send caption and photo with Quick Reply", + Text: "This is some text.", + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Tag Human Agent", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", Topic: "agent", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Send Document", + URN: "instagram:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, + SendPrep: setSendURL}, + {Label: "ID Error", + Text: "ID Error", URN: "instagram:12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, + SendPrep: setSendURL}, + {Label: "Error", + Text: "Error", URN: "instagram:12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, + SendPrep: setSendURL}, +} + func TestSending(t *testing.T) { // shorter max msg length for testing maxMsgLength = 100 - var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "2020", "US", map[string]interface{}{courier.ConfigAuthToken: "access_token"}) - RunChannelSendTestCases(t, defaultChannel, newHandler(), defaultSendTestCases, nil) + var ChannelFBA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) + var ChannelIG = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) + RunChannelSendTestCases(t, ChannelFBA, newHandler("FBA", "Facebook", false), SendTestCasesFBA, nil) + RunChannelSendTestCases(t, ChannelIG, newHandler("IG", "Instagram", false), SendTestCasesIG, nil) } func TestSigning(t *testing.T) { diff --git a/handlers/firebase/firebase.go b/handlers/firebase/firebase.go index 70e6ced82..5e9d414e8 100644 --- a/handlers/firebase/firebase.go +++ b/handlers/firebase/firebase.go @@ -117,11 +117,12 @@ func (h *handler) registerContact(ctx context.Context, channel courier.Channel, type mtPayload struct { Data struct { - Type string `json:"type"` - Title string `json:"title"` - Message string `json:"message"` - MessageID int64 `json:"message_id"` - QuickReplies []string `json:"quick_replies,omitempty"` + Type string `json:"type"` + Title string `json:"title"` + Message string `json:"message"` + MessageID int64 `json:"message_id"` + SessionStatus string `json:"session_status"` + QuickReplies []string `json:"quick_replies,omitempty"` } `json:"data"` Notification *mtNotification `json:"notification,omitempty"` ContentAvailable bool `json:"content_available"` @@ -162,6 +163,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload.Data.Title = title payload.Data.Message = part payload.Data.MessageID = int64(msg.ID()) + payload.Data.SessionStatus = msg.SessionStatus() // include any quick replies on the last piece we send if i == len(msgParts)-1 { diff --git a/handlers/firebase/firebase_test.go b/handlers/firebase/firebase_test.go index b245946d7..3d4498602 100644 --- a/handlers/firebase/firebase_test.go +++ b/handlers/firebase/firebase_test.go @@ -77,7 +77,7 @@ var notificationSendTestCases = []ChannelSendTestCase{ Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10},"notification":{"title":"FCMTitle","body":"Simple Message"},"content_available":true,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10,"session_status":""},"notification":{"title":"FCMTitle","body":"Simple Message"},"content_available":true,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, } @@ -87,22 +87,22 @@ var sendTestCases = []ChannelSendTestCase{ Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10,"session_status":""},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Long Message", Text: longMsg, - URN: "fcm:250788123123", URNAuth: "auth1", + URN: "fcm:250788123123", URNAuth: "auth1", Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"ate ac.","message_id":10},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"ate ac.","message_id":10,"session_status":""},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Quick Reply", Text: "Simple Message", URN: "fcm:250788123123", URNAuth: "auth1", QuickReplies: []string{"yes", "no"}, Attachments: []string{"image/jpeg:https://foo.bar"}, Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message\nhttps://foo.bar","message_id":10,"quick_replies":["yes","no"]},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message\nhttps://foo.bar","message_id":10,"session_status":"","quick_replies":["yes","no"]},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Error", Text: "Error", URN: "fcm:250788123123", URNAuth: "auth1", diff --git a/handlers/test.go b/handlers/test.go index 6f139d194..9f5a88e3a 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -87,7 +87,6 @@ type ChannelSendTestCase struct { QuickReplies []string Topic string HighPriority bool - ResponseToID int64 ResponseToExternalID string Metadata json.RawMessage @@ -221,7 +220,7 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour t.Run(testCase.Label, func(t *testing.T) { require := require.New(t) - msg := mb.NewOutgoingMsg(channel, courier.NewMsgID(10), urns.URN(testCase.URN), testCase.Text, testCase.HighPriority, testCase.QuickReplies, testCase.Topic, testCase.ResponseToID, testCase.ResponseToExternalID) + msg := mb.NewOutgoingMsg(channel, courier.NewMsgID(10), urns.URN(testCase.URN), testCase.Text, testCase.HighPriority, testCase.QuickReplies, testCase.Topic, testCase.ResponseToExternalID) for _, a := range testCase.Attachments { msg.WithAttachment(a) diff --git a/handlers/twiml/twiml.go b/handlers/twiml/twiml.go index c3fde453f..220a1cac5 100644 --- a/handlers/twiml/twiml.go +++ b/handlers/twiml/twiml.go @@ -79,6 +79,7 @@ type moForm struct { To string `validate:"required"` ToCountry string Body string + ButtonText string NumMedia int } @@ -138,8 +139,13 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w form.Body = handlers.DecodePossibleBase64(form.Body) } + text := form.Body + if channel.IsScheme(urns.WhatsAppScheme) && form.ButtonText != "" { + text = form.ButtonText + } + // build our msg - msg := h.Backend().NewIncomingMsg(channel, urn, form.Body).WithExternalID(form.MessageSID) + msg := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(form.MessageSID) // process any attached media for i := 0; i < form.NumMedia; i++ { diff --git a/handlers/twiml/twiml_test.go b/handlers/twiml/twiml_test.go index 19ff7d98b..9dd223e64 100644 --- a/handlers/twiml/twiml_test.go +++ b/handlers/twiml/twiml_test.go @@ -54,10 +54,11 @@ var ( twaStatusIDURL = "/c/twa/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status?id=12345" twaStatusInvalidIDURL = "/c/twa/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status?id=asdf" - receiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" - receiveMedia = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" - receiveMediaWithMsg = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&Body=Msg&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" - receiveBase64 = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=QmFubm9uIEV4cGxhaW5zIFRoZSBXb3JsZCAuLi4K4oCcVGhlIENhbXAgb2YgdGhlIFNhaW50c%2BKA&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveButtonIgnored = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&ButtonText=Confirm&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveMedia = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" + receiveMediaWithMsg = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&Body=Msg&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" + receiveBase64 = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=QmFubm9uIEV4cGxhaW5zIFRoZSBXb3JsZCAuLi4K4oCcVGhlIENhbXAgb2YgdGhlIFNhaW50c%2BKA&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" statusInvalid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=huh" statusValid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=delivered" @@ -67,6 +68,7 @@ var ( tmsReceiveExtra = "ToCountry=US&ToState=&SmsMessageSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&NumMedia=0&ToCity=&FromZip=27609&SmsSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&FromState=NC&SmsStatus=received&FromCity=RALEIGH&Body=John+Cruz&FromCountry=US&To=384387&ToZip=&NumSegments=1&MessageSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" waReceiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=whatsapp:%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=whatsapp:%2B14133881111&ApiVersion=2010-04-01" + waReceiveButtonValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&ButtonText=Confirm&FromCountry=US&To=whatsapp:%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=whatsapp:%2B14133881111&ApiVersion=2010-04-01" waReceivePrefixlessURN = "ToCountry=US&ToState=CA&SmsMessageSid=SM681a1f26d9ec591431ce406e8f399525&NumMedia=0&ToCity=&FromZip=60625&SmsSid=SM681a1f26d9ec591431ce406e8f399525&FromState=IL&SmsStatus=received&FromCity=CHICAGO&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SM681a1f26d9ec591431ce406e8f399525&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" ) @@ -74,6 +76,9 @@ var testCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: receiveURL, Data: receiveValid, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Receive Button Ignored", URL: receiveURL, Data: receiveButtonIgnored, Status: 200, Response: "", + Text: Sp("Msg"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), + PrepRequest: addValidSignature}, {Label: "Receive Invalid Signature", URL: receiveURL, Data: receiveValid, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, {Label: "Receive Missing Signature", URL: receiveURL, Data: receiveValid, Status: 400, Response: "missing request signature"}, @@ -199,6 +204,9 @@ var twaTestCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: twaReceiveURL, Data: waReceiveValid, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Receive Valid", URL: twaReceiveURL, Data: waReceiveButtonValid, Status: 200, Response: "", + Text: Sp("Confirm"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), + PrepRequest: addValidSignature}, {Label: "Receive Prefixless URN", URL: twaReceiveURL, Data: waReceivePrefixlessURN, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SM681a1f26d9ec591431ce406e8f399525"), PrepRequest: addValidSignature}, diff --git a/handlers/vk/keyboard.go b/handlers/vk/keyboard.go new file mode 100644 index 000000000..a8b435254 --- /dev/null +++ b/handlers/vk/keyboard.go @@ -0,0 +1,41 @@ +package vk + +import ( + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/jsonx" +) + +type Keyboard struct { + One_Time bool `json:"one_time"` + Buttons [][]ButtonPayload `json:"buttons"` + Inline bool `json:"inline"` +} + +type ButtonPayload struct { + Action ButtonAction `json:"action"` + Color string `json:"color"` +} + +type ButtonAction struct { + Type string `json:"type"` + Label string `json:"label"` + Payload string `json:"payload"` +} + +// NewKeyboardFromReplies creates a keyboard from the given quick replies +func NewKeyboardFromReplies(replies []string) *Keyboard { + rows := utils.StringsToRows(replies, 10, 30, 2) + buttons := make([][]ButtonPayload, len(rows)) + + for i := range rows { + buttons[i] = make([]ButtonPayload, len(rows[i])) + for j := range rows[i] { + buttons[i][j].Action.Label = rows[i][j] + buttons[i][j].Action.Type = "text" + buttons[i][j].Action.Payload = string(jsonx.MustMarshal(rows[i][j])) + buttons[i][j].Color = "primary" + } + } + + return &Keyboard{One_Time: true, Buttons: buttons, Inline: false} +} diff --git a/handlers/vk/keyboard_test.go b/handlers/vk/keyboard_test.go new file mode 100644 index 000000000..6a2491b68 --- /dev/null +++ b/handlers/vk/keyboard_test.go @@ -0,0 +1,98 @@ +package vk_test + +import ( + "testing" + + "github.com/nyaruka/courier/handlers/vk" + "github.com/stretchr/testify/assert" +) + +func TestKeyboardFromReplies(t *testing.T) { + tcs := []struct { + replies []string + expected *vk.Keyboard + }{ + { + + []string{"OK"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + { + {vk.ButtonAction{Type: "text", Label: "OK", Payload: "\"OK\""}, "primary"}, + }, + }, + false, + }, + }, + { + []string{"Yes", "No", "Maybe"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + { + {vk.ButtonAction{Type: "text", Label: "Yes", Payload: "\"Yes\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "No", Payload: "\"No\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "Maybe", Payload: "\"Maybe\""}, "primary"}, + }, + }, + false, + }, + }, + { + []string{"Vanilla", "Chocolate", "Mint", "Lemon Sorbet", "Papaya", "Strawberry"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + {{vk.ButtonAction{Type: "text", Label: "Vanilla", Payload: "\"Vanilla\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Chocolate", Payload: "\"Chocolate\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Mint", Payload: "\"Mint\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Lemon Sorbet", Payload: "\"Lemon Sorbet\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Papaya", Payload: "\"Papaya\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Strawberry", Payload: "\"Strawberry\""}, "primary"}}, + }, + false, + }, + }, + { + []string{"A", "B", "C", "D", "Chicken", "Fish", "Peanut Butter Pickle"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + {{vk.ButtonAction{Type: "text", Label: "A", Payload: "\"A\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "B", Payload: "\"B\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "C", Payload: "\"C\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "D", Payload: "\"D\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Chicken", Payload: "\"Chicken\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Fish", Payload: "\"Fish\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Peanut Butter Pickle", Payload: "\"Peanut Butter Pickle\""}, "primary"}}, + }, + false, + }, + }, + { + []string{"A", "B", "C", "D", "E"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + { + {vk.ButtonAction{Type: "text", Label: "A", Payload: "\"A\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "B", Payload: "\"B\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "C", Payload: "\"C\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "D", Payload: "\"D\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "E", Payload: "\"E\""}, "primary"}, + }, + }, + false, + }, + }, + } + + for _, tc := range tcs { + kb := vk.NewKeyboardFromReplies(tc.replies) + assert.Equal(t, tc.expected, kb, "keyboard mismatch for replies %v", tc.replies) + } +} diff --git a/handlers/vk/vk.go b/handlers/vk/vk.go index e82ceeaa3..1f39c9a85 100644 --- a/handlers/vk/vk.go +++ b/handlers/vk/vk.go @@ -19,6 +19,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/gocommon/urns" ) @@ -56,6 +57,7 @@ var ( paramMessage = "message" paramAttachments = "attachment" paramRandomId = "random_id" + paramKeyboard = "keyboard" // base upload media values paramServerId = "server" @@ -113,6 +115,7 @@ type moNewMessagePayload struct { Lng float64 `json:"longitude"` } `json:"coordinates"` } `json:"geo"` + Payload string `json:"payload"` } `json:"message" validate:"required"` } `json:"object" validate:"required"` } @@ -384,11 +387,7 @@ func takeFirstAttachmentUrl(payload moNewMessagePayload) string { func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) - if err != nil { - return status, errors.New("Cannot create send message request") - } params := buildApiBaseParams(msg.Channel()) params.Set(paramUserId, msg.URN().Path()) params.Set(paramRandomId, msg.ID().String()) @@ -397,6 +396,19 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat params.Set(paramMessage, text) params.Set(paramAttachments, attachments) + if len(msg.QuickReplies()) != 0 { + qrs := msg.QuickReplies() + keyboard := NewKeyboardFromReplies(qrs) + + params.Set(paramKeyboard, string(jsonx.MustMarshal(keyboard))) + } + + req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) + + if err != nil { + return status, errors.New("Cannot create send message request") + } + req.URL.RawQuery = params.Encode() res, err := utils.MakeHTTPRequest(req) diff --git a/handlers/vk/vk_test.go b/handlers/vk/vk_test.go index 01cef3e58..980755132 100644 --- a/handlers/vk/vk_test.go +++ b/handlers/vk/vk_test.go @@ -2,8 +2,6 @@ package vk import ( "context" - "github.com/nyaruka/gocommon/urns" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "net/url" @@ -11,6 +9,9 @@ import ( "testing" "time" + "github.com/nyaruka/gocommon/urns" + "github.com/stretchr/testify/assert" + "github.com/nyaruka/courier" . "github.com/nyaruka/courier/handlers" ) @@ -210,6 +211,22 @@ const eventServerVerification = `{ "secret": "abc123xyz" }` +const msgKeyboard = `{ + "type": "message_new", + "object": { + "message": { + "id": 1, + "date": 1580125800, + "from_id": 123456, + "text": "Yes", + "payload": "\"Yes\"" + } + }, + "secret": "abc123xyz" + }` + +const keyboardJson = `{"one_time":true,"buttons":[[{"action":{"type":"text","label":"A","payload":"\"A\""},"color":"primary"},{"action":{"type":"text","label":"B","payload":"\"B\""},"color":"primary"},{"action":{"type":"text","label":"C","payload":"\"C\""},"color":"primary"},{"action":{"type":"text","label":"D","payload":"\"D\""},"color":"primary"},{"action":{"type":"text","label":"E","payload":"\"E\""},"color":"primary"}]],"inline":false}` + var testCases = []ChannelHandleTestCase{ { Label: "Receive Message", @@ -281,6 +298,16 @@ var testCases = []ChannelHandleTestCase{ ExternalID: Sp("1"), Date: Tp(time.Date(2020, 1, 27, 11, 50, 0, 0, time.UTC)), Attachments: []string{"https://foo.bar/doc.pdf"}, }, + { + Label: "Receive Message Keyboard", + URL: receiveURL, + Data: msgKeyboard, + Status: 200, + Response: "ok", + URN: Sp("vk:123456"), + ExternalID: Sp("1"), + Date: Tp(time.Date(2020, 1, 27, 11, 50, 0, 0, time.UTC)), + }, { Label: "Receive Geolocation Attachment", URL: receiveURL, @@ -445,6 +472,25 @@ var sendTestCases = []ChannelSendTestCase{ }, }, }, + { + Label: "Send keyboard", + Text: "Send keyboard", + URN: "vk:123456789", + QuickReplies: []string{"A", "B", "C", "D", "E"}, + Status: "S", + SendPrep: setSendURL, + ExternalID: "1", + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: actionSendMessage, + RawQuery: "access_token=token123xyz&attachment=&keyboard=" + url.QueryEscape(keyboardJson) + "&message=Send+keyboard&random_id=10&user_id=123456789&v=5.103", + }: { + Status: 200, + Body: `{"response": 1}`, + }, + }, + }, } func mockAttachmentURLs(mediaServer *httptest.Server, testCases []ChannelSendTestCase) []ChannelSendTestCase { diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 102b6307e..14d9b02a7 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -12,12 +12,13 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/backends/rapidpro" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" - "github.com/nyaruka/gocommon/rcache" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/redisx" "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -557,6 +558,8 @@ const maxMsgLength = 4096 // SendMsg sends the passed in message, returning any error func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { start := time.Now() + conn := h.Backend().RedisPool().Get() + defer conn.Close() // get our token token := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") @@ -588,8 +591,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat for i, payload := range payloads { externalID := "" - - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(conn, msg, sendPath, payload) // add logs to our status for _, log := range logs { status.AddLog(log) @@ -618,7 +620,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } status.SetStatus(courier.MsgWired) } - return status, nil } @@ -628,169 +629,108 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann var logs []*courier.ChannelLog var err error - // do we have a template? + //do we have a template? templating, err := h.getTemplate(msg) - if templating != nil || len(msg.Attachments()) == 0 { - - if err != nil { + if err != nil { return nil, nil, errors.Wrapf(err, "unable to decode template: %s for channel: %s", string(msg.Metadata()), msg.Channel().UUID()) + } + + if templating != nil { + + namespace := templating.Namespace + if namespace == "" { + namespace = msg.Channel().StringConfigForKey(configNamespace, "") + } + if namespace == "" { + return nil, nil, errors.Errorf("cannot send template message without Facebook namespace for channel: %s", msg.Channel().UUID()) } - if templating != nil { - namespace := templating.Namespace - if namespace == "" { - namespace = msg.Channel().StringConfigForKey(configNamespace, "") + + if msg.Channel().BoolConfigForKey(configHSMSupport, false) { + payload := hsmPayload{ + To: msg.URN().Path(), + Type: "hsm", } - if namespace == "" { - return nil, nil, errors.Errorf("cannot send template message without Facebook namespace for channel: %s", msg.Channel().UUID()) + payload.HSM.Namespace = namespace + payload.HSM.ElementName = templating.Template.Name + payload.HSM.Language.Policy = "deterministic" + payload.HSM.Language.Code = templating.Language + for _, v := range templating.Variables { + payload.HSM.LocalizableParams = append(payload.HSM.LocalizableParams, LocalizableParam{Default: v}) } + payloads = append(payloads, payload) + } else { + payload := templatePayload{ + To: msg.URN().Path(), + Type: "template", + } + payload.Template.Namespace = namespace + payload.Template.Name = templating.Template.Name + payload.Template.Language.Policy = "deterministic" + payload.Template.Language.Code = templating.Language - if msg.Channel().BoolConfigForKey(configHSMSupport, false) { - payload := hsmPayload{ - To: msg.URN().Path(), - Type: "hsm", - } - payload.HSM.Namespace = namespace - payload.HSM.ElementName = templating.Template.Name - payload.HSM.Language.Policy = "deterministic" - payload.HSM.Language.Code = templating.Language - for _, v := range templating.Variables { - payload.HSM.LocalizableParams = append(payload.HSM.LocalizableParams, LocalizableParam{Default: v}) - } - payloads = append(payloads, payload) - } else { - payload := templatePayload{ - To: msg.URN().Path(), - Type: "template", - } - payload.Template.Namespace = namespace - payload.Template.Name = templating.Template.Name - payload.Template.Language.Policy = "deterministic" - payload.Template.Language.Code = templating.Language - - component := &Component{Type: "body"} + component := &Component{Type: "body"} - for _, v := range templating.Variables { - component.Parameters = append(component.Parameters, Param{Type: "text", Text: v}) - } - payload.Template.Components = append(payload.Template.Components, *component) + for _, v := range templating.Variables { + component.Parameters = append(component.Parameters, Param{Type: "text", Text: v}) + } + payload.Template.Components = append(payload.Template.Components, *component) - if len(msg.Attachments()) > 0 { + if len(msg.Attachments()) > 0 { - header := &Component{Type: "header"} + header := &Component{Type: "header"} - for _, attachment := range msg.Attachments() { + for _, attachment := range msg.Attachments() { - mimeType, mediaURL := handlers.SplitAttachment(attachment) - mediaID, mediaLogs, err := h.fetchMediaID(msg, mimeType, mediaURL) - if len(mediaLogs) > 0 { - logs = append(logs, mediaLogs...) - } - if err != nil { - logrus.WithField("channel_uuid", msg.Channel().UUID().String()).WithError(err).Error("error while uploading media to whatsapp") - } - if err != nil && mediaID != "" { - mediaURL = "" - } - if strings.HasPrefix(mimeType, "image") { - image := &mmtImage{ - Link: mediaURL, - } - header.Parameters = append(header.Parameters, Param{Type: "image", Image: image}) - payload.Template.Components = append(payload.Template.Components, *header) - } else if strings.HasPrefix(mimeType, "application") { - document := &mmtDocument{ - Link: mediaURL, - } - header.Parameters = append(header.Parameters, Param{Type: "document", Document: document}) - payload.Template.Components = append(payload.Template.Components, *header) - } else if strings.HasPrefix(mimeType, "video") { - video := &mmtVideo{ - Link: mediaURL, - } - header.Parameters = append(header.Parameters, Param{Type: "video", Video: video}) - payload.Template.Components = append(payload.Template.Components, *header) - } else { - duration := time.Since(start) - err = fmt.Errorf("unknown attachment mime type: %s", mimeType) - attachmentLogs := []*courier.ChannelLog{courier.NewChannelLogFromError("Error sending message", msg.Channel(), msg.ID(), duration, err)} - logs = append(logs, attachmentLogs...) - break - } + mimeType, mediaURL := handlers.SplitAttachment(attachment) + mediaID, mediaLogs, err := h.fetchMediaID(msg, mimeType, mediaURL) + if len(mediaLogs) > 0 { + logs = append(logs, mediaLogs...) } - } - payloads = append(payloads, payload) - } - } else { - parts := handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) - - qrs := msg.QuickReplies() - wppVersion := msg.Channel().ConfigForKey("version", "0").(string) - isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) - isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) - - if isInteractiveMsg { - for i, part := range parts { - if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive - payload := mtTextPayload{ - To: msg.URN().Path(), - Type: "text", + if err != nil { + logrus.WithField("channel_uuid", msg.Channel().UUID().String()).WithError(err).Error("error while uploading media to whatsapp") + } + if err != nil && mediaID != "" { + mediaURL = "" + } + if strings.HasPrefix(mimeType, "image") { + image := &mmtImage{ + Link: mediaURL, } - payload.Text.Body = part - payloads = append(payloads, payload) - - } else { - payload := mtInteractivePayload{ - To: msg.URN().Path(), - Type: "interactive", + header.Parameters = append(header.Parameters, Param{Type: "image", Image: image}) + payload.Template.Components = append(payload.Template.Components, *header) + } else if strings.HasPrefix(mimeType, "application") { + document := &mmtDocument{ + Link: mediaURL, } - - // up to 3 qrs the interactive message will be button type, otherwise it will be list - if len(qrs) <= 3 { - payload.Interactive.Type = "button" - payload.Interactive.Body.Text = part - btns := make([]mtButton, len(qrs)) - for i, qr := range qrs { - btns[i] = mtButton{ - Type: "reply", - } - btns[i].Reply.ID = fmt.Sprint(i) - btns[i].Reply.Title = qr - } - payload.Interactive.Action.Buttons = btns - payloads = append(payloads, payload) - } else { - payload.Interactive.Type = "list" - payload.Interactive.Body.Text = part - payload.Interactive.Action.Button = "Menu" - section := mtSection{ - Rows: make([]mtSectionRow, len(qrs)), - } - for i, qr := range qrs { - section.Rows[i] = mtSectionRow{ - ID: fmt.Sprint(i), - Title: qr, - } - } - payload.Interactive.Action.Sections = []mtSection{ - section, - } - payloads = append(payloads, payload) + header.Parameters = append(header.Parameters, Param{Type: "document", Document: document}) + payload.Template.Components = append(payload.Template.Components, *header) + } else if strings.HasPrefix(mimeType, "video") { + video := &mmtVideo{ + Link: mediaURL, } + header.Parameters = append(header.Parameters, Param{Type: "video", Video: video}) + payload.Template.Components = append(payload.Template.Components, *header) + } else { + duration := time.Since(start) + err = fmt.Errorf("unknown attachment mime type: %s", mimeType) + attachmentLogs := []*courier.ChannelLog{courier.NewChannelLogFromError("Error sending message", msg.Channel(), msg.ID(), duration, err)} + logs = append(logs, attachmentLogs...) + break } } - } else { - for _, part := range parts { - payload := mtTextPayload{ - To: msg.URN().Path(), - Type: "text", - } - payload.Text.Body = part - payloads = append(payloads, payload) - } } + payloads = append(payloads, payload) } + } else { + parts := handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + + qrs := msg.QuickReplies() + wppVersion := msg.Channel().ConfigForKey("version", "0").(string) + isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) + isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) + if len(msg.Attachments()) > 0 { for attachmentCount, attachment := range msg.Attachments() { @@ -818,7 +758,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "document", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } mediaPayload.Filename, err = utils.BasePathForURL(mediaURL) @@ -834,7 +774,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "image", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } payload.Image = mediaPayload @@ -844,7 +784,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "video", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } payload.Video = mediaPayload @@ -857,8 +797,123 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann break } } + + if isInteractiveMsg { + for i, part := range parts { + if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive + payload := mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + payload.Text.Body = part + payloads = append(payloads, payload) + + } else { + payload := mtInteractivePayload{ + To: msg.URN().Path(), + Type: "interactive", + } + + // up to 3 qrs the interactive message will be button type, otherwise it will be list + if len(qrs) <= 3 { + payload.Interactive.Type = "button" + payload.Interactive.Body.Text = part + btns := make([]mtButton, len(qrs)) + for i, qr := range qrs { + btns[i] = mtButton{ + Type: "reply", + } + btns[i].Reply.ID = fmt.Sprint(i) + btns[i].Reply.Title = qr + } + payload.Interactive.Action.Buttons = btns + payloads = append(payloads, payload) + } else { + payload.Interactive.Type = "list" + payload.Interactive.Body.Text = part + payload.Interactive.Action.Button = "Menu" + section := mtSection{ + Rows: make([]mtSectionRow, len(qrs)), + } + for i, qr := range qrs { + section.Rows[i] = mtSectionRow{ + ID: fmt.Sprint(i), + Title: qr, + } + } + payload.Interactive.Action.Sections = []mtSection{ + section, + } + payloads = append(payloads, payload) + } + } + } + } + } else { + if isInteractiveMsg { + for i, part := range parts { + if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive + payload := mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + payload.Text.Body = part + payloads = append(payloads, payload) + + } else { + payload := mtInteractivePayload{ + To: msg.URN().Path(), + Type: "interactive", + } + + // up to 3 qrs the interactive message will be button type, otherwise it will be list + if len(qrs) <= 3 { + payload.Interactive.Type = "button" + payload.Interactive.Body.Text = part + btns := make([]mtButton, len(qrs)) + for i, qr := range qrs { + btns[i] = mtButton{ + Type: "reply", + } + btns[i].Reply.ID = fmt.Sprint(i) + btns[i].Reply.Title = qr + } + payload.Interactive.Action.Buttons = btns + payloads = append(payloads, payload) + } else { + payload.Interactive.Type = "list" + payload.Interactive.Body.Text = part + payload.Interactive.Action.Button = "Menu" + section := mtSection{ + Rows: make([]mtSectionRow, len(qrs)), + } + for i, qr := range qrs { + section.Rows[i] = mtSectionRow{ + ID: fmt.Sprint(i), + Title: qr, + } + } + payload.Interactive.Action.Sections = []mtSection{ + section, + } + payloads = append(payloads, payload) + } + } + } + } else { + for _, part := range parts { + payload := mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + payload.Text.Body = part + payloads = append(payloads, payload) + } + } + } - } + + } return payloads, logs, err } @@ -871,7 +926,8 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri defer rc.Close() cacheKey := fmt.Sprintf(mediaCacheKeyPattern, msg.Channel().UUID().String()) - mediaID, err := rcache.Get(rc, cacheKey, mediaURL) + mediaCache := redisx.NewIntervalHash(cacheKey, time.Hour*24, 2) + mediaID, err := mediaCache.Get(rc, mediaURL) if err != nil { return "", logs, errors.Wrapf(err, "error reading media id from redis: %s : %s", cacheKey, mediaURL) } else if mediaID != "" { @@ -929,7 +985,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri } // put in cache - err = rcache.Set(rc, cacheKey, mediaURL, mediaID) + err = mediaCache.Set(rc, mediaURL, mediaID) if err != nil { return "", logs, errors.Wrapf(err, "error setting media id in cache") } @@ -937,7 +993,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return mediaID, logs, nil } -func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { +func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { start := time.Now() jsonBody, err := json.Marshal(payload) @@ -950,6 +1006,20 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (s req, _ := http.NewRequest(http.MethodPost, sendPath.String(), bytes.NewReader(jsonBody)) req.Header = buildWhatsAppHeaders(msg.Channel()) rr, err := utils.MakeHTTPRequest(req) + + if rr.StatusCode == 429 || rr.StatusCode == 503 { + rateLimitKey := fmt.Sprintf("rate_limit:%s", msg.Channel().UUID().String()) + rc.Do("set", rateLimitKey, "engaged") + + // The rate limit is 50 requests per second + // We pause sending 2 seconds so the limit count is reset + // TODO: In the future we should the header value when available + rc.Do("expire", rateLimitKey, 2) + + log := courier.NewChannelLogFromRR("rate limit engaged", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + return "", "", []*courier.ChannelLog{log}, err + } + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) errPayload := &mtErrorPayload{} err = json.Unmarshal(rr.Body, errPayload) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index f7ef157d1..b0b2e4e6f 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -443,6 +443,12 @@ var defaultSendTestCases = []ChannelSendTestCase{ ResponseBody: `{ "errors": [{ "title": "Error Sending" }] }`, ResponseStatus: 403, RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Error"}}`, SendPrep: setSendURL}, + {Label: "Rate Limit Engaged", + Text: "Error", URN: "whatsapp:250788123123", + Status: "E", + ResponseBody: `{ "errors": [{ "title": "Too many requests" }] }`, ResponseStatus: 429, + RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Error"}}`, + SendPrep: setSendURL}, {Label: "No Message ID", Text: "Error", URN: "whatsapp:250788123123", Status: "E", @@ -678,6 +684,52 @@ var defaultSendTestCases = []ChannelSendTestCase{ ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"template","template":{"namespace":"wa_template_namespace","name":"revive_issue","language":{"policy":"deterministic","code":"en_US"},"components":[{"type":"body","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]},{"type":"header","parameters":[{"type":"document","document":{"link":"https://foo.bar/document.pdf"}}]}]}}`, SendPrep: setSendURL}, + {Label: "Interactive Button Message Send with attachment", + Text: "Interactive Button Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"BUTTON1"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"interactive","interactive":{"type":"button","body":{"text":"Interactive Button Msg"},"action":{"buttons":[{"type":"reply","reply":{"id":"0","title":"BUTTON1"}}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + {Label: "Interactive List Message Send with attachment", + Text: "Interactive List Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"ROW1", "ROW2", "ROW3", "ROW4"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, } var mediaCacheSendTestCases = []ChannelSendTestCase{ diff --git a/msg.go b/msg.go index 7ee4044e9..809370f20 100644 --- a/msg.go +++ b/msg.go @@ -95,7 +95,6 @@ type Msg interface { QuickReplies() []string Topic() string Metadata() json.RawMessage - ResponseToID() MsgID ResponseToExternalID() string IsResend() bool diff --git a/queue/queue.go b/queue/queue.go index 7aa4bebaa..da6b96e83 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -83,10 +83,24 @@ var luaPop = redis.NewScript(2, `-- KEYS: [EpochMS QueueType] local delim = string.find(queue, "|") local tps = 0 local tpsKey = "" + + local queueName = "" + if delim then + queueName = string.sub(queue, string.len(KEYS[2])+2, delim-1) tps = tonumber(string.sub(queue, delim+1)) end + if queueName then + local rateLimitKey = "rate_limit:" .. queueName + local rateLimitEngaged = redis.call("get", rateLimitKey) + if rateLimitEngaged then + redis.call("zincrby", KEYS[2] .. ":throttled", workers, queue) + redis.call("zrem", KEYS[2] .. ":active", queue) + return {"retry", ""} + end + end + -- if we have a tps, then check whether we exceed it if tps > 0 then tpsKey = queue .. ":tps:" .. math.floor(KEYS[1]) diff --git a/queue/queue_test.go b/queue/queue_test.go index 157793a0e..81cd415a3 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -50,6 +50,7 @@ func TestLua(t *testing.T) { defer close(quitter) rate := 10 + for i := 0; i < 20; i++ { err := PushOntoQueue(conn, "msgs", "chan1", rate, fmt.Sprintf(`[{"id":%d}]`, i), LowPriority) assert.NoError(err) @@ -166,6 +167,51 @@ func TestLua(t *testing.T) { assert.NoError(err) assert.Equal(EmptyQueue, queue) assert.Empty(value) + + err = PushOntoQueue(conn, "msgs", "chan1", rate, `[{"id":34}]`, HighPriority) + assert.NoError(err) + + conn.Do("set", "rate_limit:chan1", "engaged") + conn.Do("EXPIRE", "rate_limit:chan1", 5) + + // we have the rate limit set, + queue, value, err = PopFromQueue(conn, "msgs") + if value != "" && queue != EmptyQueue { + t.Fatal("Should be throttled") + } + + time.Sleep(2 * time.Second) + queue, value, err = PopFromQueue(conn, "msgs") + if value != "" && queue != EmptyQueue { + t.Fatal("Should be throttled") + } + + count, err = redis.Int(conn.Do("zcard", "msgs:throttled")) + assert.NoError(err) + assert.Equal(1, count, "Expected chan1 to be throttled") + + count, err = redis.Int(conn.Do("zcard", "msgs:active")) + assert.NoError(err) + assert.Equal(0, count, "Expected chan1 to not be active") + + // but if we wait for the rate limit to expire + time.Sleep(3 * time.Second) + + // next should be 34 + queue, value, err = PopFromQueue(conn, "msgs") + assert.NotEqual(queue, EmptyQueue) + assert.Equal(`{"id":34}`, value) + assert.NoError(err) + + // nothing should be left + queue = Retry + for queue == Retry { + queue, value, err = PopFromQueue(conn, "msgs") + } + assert.NoError(err) + assert.Equal(EmptyQueue, queue) + assert.Empty(value) + } func nTestThrottle(t *testing.T) { diff --git a/sender.go b/sender.go index 63d966822..dbed21aab 100644 --- a/sender.go +++ b/sender.go @@ -189,23 +189,10 @@ func (w *Sender) sendMessage(msg Msg) { log.WithError(err).Error("error looking up msg was sent") } - // is this msg in a loop? - loop, err := backend.IsMsgLoop(sendCTX, msg) - - // failing on loop lookup isn't permanent, but log - if err != nil { - log.WithError(err).Error("error looking up msg loop") - } - if sent { // if this message was already sent, create a wired status for it status = backend.NewMsgStatusForID(msg.Channel(), msg.ID(), MsgWired) log.Warning("duplicate send, marking as wired") - } else if loop { - // if this contact is in a loop, fail the message immediately without sending - status = backend.NewMsgStatusForID(msg.Channel(), msg.ID(), MsgFailed) - status.AddLog(NewChannelLogFromError("Message Loop", msg.Channel(), msg.ID(), 0, fmt.Errorf("message loop detected, failing message without send"))) - log.Error("message loop detected, failing message") } else { // send our message status, err = server.SendMsg(sendCTX, msg) diff --git a/test.go b/test.go index 527853648..012c67a1b 100644 --- a/test.go +++ b/test.go @@ -119,13 +119,8 @@ func (mb *MockBackend) NewIncomingMsg(channel Channel, urn urns.URN, text string } // NewOutgoingMsg creates a new outgoing message from the given params -func (mb *MockBackend) NewOutgoingMsg(channel Channel, id MsgID, urn urns.URN, text string, highPriority bool, quickReplies []string, topic string, responseToID int64, responseToExternalID string) Msg { - msgResponseToID := NilMsgID - if responseToID != 0 { - msgResponseToID = NewMsgID(responseToID) - } - - return &mockMsg{channel: channel, id: id, urn: urn, text: text, highPriority: highPriority, quickReplies: quickReplies, topic: topic, responseToID: msgResponseToID, responseToExternalID: responseToExternalID} +func (mb *MockBackend) NewOutgoingMsg(channel Channel, id MsgID, urn urns.URN, text string, highPriority bool, quickReplies []string, topic string, responseToExternalID string) Msg { + return &mockMsg{channel: channel, id: id, urn: urn, text: text, highPriority: highPriority, quickReplies: quickReplies, topic: topic, responseToExternalID: responseToExternalID} } // PushOutgoingMsg is a test method to add a message to our queue of messages to send @@ -166,11 +161,6 @@ func (mb *MockBackend) ClearMsgSent(ctx context.Context, id MsgID) error { return nil } -// IsMsgLoop returns whether the passed in msg is a loop -func (mb *MockBackend) IsMsgLoop(ctx context.Context, msg Msg) (bool, error) { - return false, nil -} - // MarkOutgoingMsgComplete marks the passed msg as having been dealt with func (mb *MockBackend) MarkOutgoingMsgComplete(ctx context.Context, msg Msg, s MsgStatus) { mb.mutex.Lock() @@ -568,7 +558,6 @@ type mockMsg struct { highPriority bool quickReplies []string topic string - responseToID MsgID responseToExternalID string metadata json.RawMessage alreadyWritten bool @@ -594,7 +583,6 @@ func (m *mockMsg) ContactName() string { return m.contactName } func (m *mockMsg) HighPriority() bool { return m.highPriority } func (m *mockMsg) QuickReplies() []string { return m.quickReplies } func (m *mockMsg) Topic() string { return m.topic } -func (m *mockMsg) ResponseToID() MsgID { return m.responseToID } func (m *mockMsg) ResponseToExternalID() string { return m.responseToExternalID } func (m *mockMsg) Metadata() json.RawMessage { return m.metadata } func (m *mockMsg) IsResend() bool { return m.isResend } diff --git a/utils/misc.go b/utils/misc.go index 222b00251..8f73b9a15 100644 --- a/utils/misc.go +++ b/utils/misc.go @@ -5,7 +5,6 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - "encoding/json" "net/url" "path" "regexp" @@ -22,15 +21,6 @@ func SignHMAC256(privateKey string, value string) string { return signedParams } -// MapAsJSON serializes the given map as a JSON string -func MapAsJSON(m map[string]string) []byte { - bytes, err := json.Marshal(m) - if err != nil { - panic(err) - } - return bytes -} - // JoinNonEmpty takes a vararg of strings and return the join of all the non-empty strings with a delimiter between them func JoinNonEmpty(delim string, strings ...string) string { var buf bytes.Buffer diff --git a/utils/misc_test.go b/utils/misc_test.go index 23c30c26d..6e760b825 100644 --- a/utils/misc_test.go +++ b/utils/misc_test.go @@ -14,11 +14,6 @@ func TestSignHMAC256(t *testing.T) { assert.Len(t, utils.SignHMAC256("ZXwAumfRSejDxJGa", "newValueToEncrypt"), 64) } -func TestMapAsJSON(t *testing.T) { - assert.Equal(t, "{}", string(utils.MapAsJSON(map[string]string{}))) - assert.Equal(t, "{\"foo\":\"bar\"}", string(utils.MapAsJSON(map[string]string{"foo": "bar"}))) -} - func TestJoinNonEmpty(t *testing.T) { assert.Equal(t, "", utils.JoinNonEmpty(" ")) assert.Equal(t, "hello world", utils.JoinNonEmpty(" ", "", "hello", "", "world"))