Merge pull request #165 from Ilhasoft/update/v9.0.1

Update to v9.0.1

.github/workflows/build-courier-push-tag-sp-india-ire.yaml

@@ -16,7 +16,6 @@ jobs:
             if grep -qs -e '^.*.*-develop' <<< "${TAG}" ; then
               echo "Found environment: DEVELOP - ${TAG}"
               echo "ENVIRONMENT=develop" | tee -a "${GITHUB_ENV}"
-              exit 1 # stop action
             elif grep -qs -e '^.*.*-staging' <<< "${TAG}" ; then
               echo "Found environment: STAGING - ${TAG}"
               echo "ENVIRONMENT=staging" | tee -a "${GITHUB_ENV}"

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: "1.19.x"
+  go-version: "1.21.x"
 jobs:
   test:
     name: Test
@@ -13,7 +13,7 @@ jobs:
         ports:
           - 6379:6379
       postgres:
-        image: postgres:14-alpine
+        image: postgres:15-alpine
         env:
           POSTGRES_DB: courier_test
           POSTGRES_USER: courier_test

CHANGELOG.md

@@ -1,3 +1,171 @@
+v9.0.1 (2024-01-08)
+-------------------------
+ * Fix sending bandwidth MMS without text
+
+v9.0.0 (2024-01-05)
+-------------------------
+ * Bump golang.org/x/crypto from 0.16.0 to 0.17.0
+
+v8.3.32 (2023-12-12)
+-------------------------
+ * Update deps
+
+v8.3.31 (2023-12-06)
+-------------------------
+ * Use language value in templating metadata instead of trying to match
+
+v8.3.30 (2023-12-04)
+-------------------------
+ * Change channel events so that created_on is db time and is included in queued task payload
+
+v8.3.29 (2023-12-04)
+-------------------------
+ * Fix FBA timestamps that sometimes are in seconds instead of milliseconds
+ * Remove support for HSM template support
+ * Update to latest gocommon and phonenumbers
+
+v8.3.28 (2023-11-23)
+-------------------------
+ * Logging tweak
+
+v8.3.27 (2023-10-31)
+-------------------------
+ * Prevent all courier HTTP requests from accessing local networks
+
+v8.3.26 (2023-10-30)
+-------------------------
+ * Update to latest gocommon
+
+v8.3.25 (2023-10-25)
+-------------------------
+ * Update docker image to go 1.21
+ * Remove use of logrus and use slog with sentry
+ * Bump golang.org/x/net from 0.14.0 to 0.17.0
+
+v8.3.24 (2023-10-10)
+-------------------------
+ * Fix handling IG like hearts
+ * Ignore attachments of type fallback on FBA channels
+ * More logrus replacement to use slog
+
+v8.3.23 (2023-10-04)
+-------------------------
+ * Switch channelevent.extra to always be strings
+ * Add optin_id to channels_channelevent
+ * Allow outgoing tests to check multiple requests
+
+v8.3.22 (2023-09-27)
+-------------------------
+ * Use Facebook API v17.0
+
+v8.3.21 (2023-09-25)
+-------------------------
+ * Support sending facebook message with opt-in auth token
+
+v8.3.20 (2023-09-21)
+-------------------------
+ * Switch to using optin ids instead of uuids
+
+v8.3.19 (2023-09-20)
+-------------------------
+ * Fix queueing of optin/optout events to mailroom
+ * Implement sending opt-in requests for FBA channels
+ * Simplfy handlers splitting up messages
+
+v8.3.18 (2023-09-18)
+-------------------------
+ * Add separate MsgIn and MsgOut interface types
+ * Use functional options pattern to create base handlers
+ * Improve testing of status updates from handlers and allow testing of multiple status updates per request
+ * Split up Meta notification payload into whatsapp and messenger specific parts
+
+v8.3.17 (2023-09-14)
+-------------------------
+ * Fix stop contact event task names
+ * Add support for FB notificaiton messages optin and optout events
+
+v8.3.16 (2023-09-13)
+-------------------------
+ * Simplify interfaces that handlers have access to
+ * Allow handlers to create arbitrary auth tokens with messages and channel events
+ * Rename legacy FB and WA handlers
+ * Refactor whatsapp handlers to be more DRY
+
+v8.3.15 (2023-09-12)
+-------------------------
+ * Stop reading from ContactURN.auth and remove from model
+
+v8.3.14 (2023-09-11)
+-------------------------
+ * Move whatsapp language matching into own util package and use i18n.BCP47Matcher
+ * Update to latest gocommon and use i18n.Locale
+ * Read from ContactURN.auth_tokens instead of .auth
+
+v8.3.13 (2023-09-06)
+-------------------------
+ * Start writing ContactURN.auth_tokens
+ * Update to latest null library and use Map[string] for channel event extra
+
+v8.3.12 (2023-09-06)
+-------------------------
+ * Do more debug logging and less info logging
+
+v8.3.11 (2023-09-06)
+-------------------------
+ * Add logging of requests with no associated channel
+ * No need to try making DB queries when all msg IDs got resolved from redis
+
+v8.3.10 (2023-09-05)
+-------------------------
+ * Don't rely on knowing msg id to determine if a log is attached
+ * Rework handler tests so that test cases must explicitly say if they don't generate a channel log
+
+v8.3.9 (2023-09-05)
+-------------------------
+ * Try to resolve sent external ids from redis
+ * For received messages without external id, de-dupe by hash of text+attachments instead of just text
+
+v8.3.8 (2023-08-31)
+-------------------------
+ * Update to latest redisx which fixes accuracy for sub-minute interval hashes
+ * Update to new batchers in gocommon which are more efficient
+
+v8.3.7 (2023-08-30)
+-------------------------
+ * Sender deletion handled by mailroom task
+
+v8.3.6 (2023-08-30)
+-------------------------
+ * Rework writing msg statuses to always use id resolving
+
+v8.3.5 (2023-08-30)
+-------------------------
+ * Rework writing status updates so that updates by external id also use the batcher
+
+v8.3.4 (2023-08-24)
+-------------------------
+ * Update channel type to save external ID for MO messages if we can, so we can dedupe by that
+ * Test with PostgreSQL 15
+
+v8.3.3 (2023-08-17)
+-------------------------
+ * Remove Legacy Twitter (TT) type registration
+ * Remove Blackmyna, Junebug, old Zenvia channel type handlers
+
+v8.3.2 (2023-08-16)
+-------------------------
+ * Fix retrieve media files for D3C
+
+v8.3.1 (2023-08-09)
+-------------------------
+ * Revert validator dep upgrade
+
+v8.3.0 (2023-08-09)
+-------------------------
+ * Update to go 1.20
+ * Update deps
+ * Add Messagebird channel type
+
 v8.2.1 (2023-08-03)
 -------------------------
  * Always save http_logs as [] rather than null

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20
+FROM golang:1.21
 
 WORKDIR /usr/src/app
 

README.md

@@ -35,7 +35,7 @@ environment variables and parameters and for more details on each option.
 
 ### RapidPro
 
-For use with RapidPro, you will want to configure these settings:
+For use with RapidPro/TextIt, you will want to configure these settings:
 
  * `COURIER_DOMAIN`: The root domain which courier is exposed as (ex `textit.in`)
  * `COURIER_SPOOL_DIR`: A local path where courier can spool files if the database is down, should be writable. (ex: `/home/courier/spool`)

WENI-CHANGELOG.md

@@ -1,3 +1,8 @@
+1.5.9-courier-9.0.1
+----------
+  * Update to v9.0.1
+  * Update dockerfile
+
 1.5.8-courier-8.2.1
 ----------
   * Remove menu button name mapping

attachments.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"log/slog"
 	"mime"
 	"net/http"
 	"net/url"
@@ -12,7 +13,6 @@ import (
 	"github.com/nyaruka/courier/utils"
 	"github.com/nyaruka/gocommon/httpx"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"gopkg.in/h2non/filetype.v1"
 )
 
@@ -57,14 +57,14 @@ func fetchAttachment(ctx context.Context, b Backend, r *http.Request) (*fetchAtt
 		return nil, errors.Wrap(err, "error getting channel")
 	}
 
-	clog := NewChannelLogForAttachmentFetch(ch, fa.MsgID, GetHandler(ch.ChannelType()).RedactValues(ch))
+	clog := NewChannelLogForAttachmentFetch(ch, GetHandler(ch.ChannelType()).RedactValues(ch))
 
 	attachment, err := FetchAndStoreAttachment(ctx, b, ch, fa.URL, clog)
 
 	// try to write channel log even if we have an error
 	clog.End()
 	if err := b.WriteChannelLog(ctx, clog); err != nil {
-		logrus.WithError(err).Error()
+		slog.Error("error writing log", "error", err)
 	}
 
 	if err != nil {
@@ -93,13 +93,13 @@ func FetchAndStoreAttachment(ctx context.Context, b Backend, channel Channel, at
 		return nil, errors.Wrap(err, "unable to create attachment request")
 	}
 
-	trace, err := httpx.DoTrace(utils.GetHTTPClient(), attRequest, nil, nil, maxAttBodyReadBytes)
+	trace, err := httpx.DoTrace(b.HttpClient(true), attRequest, nil, b.HttpAccess(), maxAttBodyReadBytes)
 	if trace != nil {
 		clog.HTTP(trace)
 
 		// if we got a non-200 response, return the attachment with a pseudo content type which tells the caller
 		// to continue without the attachment
-		if trace.Response == nil || trace.Response.StatusCode/100 != 2 {
+		if trace.Response == nil || trace.Response.StatusCode/100 != 2 || err == httpx.ErrResponseSize || err == httpx.ErrAccessConfig {
 			return &Attachment{ContentType: "unavailable", URL: attURL}, nil
 		}
 	}

attachments_test.go

@@ -37,10 +37,10 @@ func TestFetchAndStoreAttachment(t *testing.T) {
 	ctx := context.Background()
 	mb := test.NewMockBackend()
 
-	mockChannel := test.NewMockChannel("e4bb1578-29da-4fa5-a214-9da19dd24230", "MCK", "2020", "US", map[string]interface{}{})
+	mockChannel := test.NewMockChannel("e4bb1578-29da-4fa5-a214-9da19dd24230", "MCK", "2020", "US", map[string]any{})
 	mb.AddChannel(mockChannel)
 
-	clog := courier.NewChannelLogForAttachmentFetch(mockChannel, courier.MsgID(123), []string{"sesame"})
+	clog := courier.NewChannelLogForAttachmentFetch(mockChannel, []string{"sesame"})
 
 	att, err := courier.FetchAndStoreAttachment(ctx, mb, mockChannel, "http://mock.com/media/hello.jpg", clog)
 	assert.NoError(t, err)

backend.go

@@ -3,9 +3,11 @@ package courier
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"strings"
 
 	"github.com/gomodule/redigo/redis"
+	"github.com/nyaruka/gocommon/httpx"
 	"github.com/nyaruka/gocommon/urns"
 )
 
@@ -30,31 +32,31 @@ type Backend interface {
 	GetChannelByAddress(context.Context, ChannelType, ChannelAddress) (Channel, error)
 
 	// GetContact returns (or creates) the contact for the passed in channel and URN
-	GetContact(context.Context, Channel, urns.URN, string, string, *ChannelLog) (Contact, error)
+	GetContact(context.Context, Channel, urns.URN, map[string]string, string, *ChannelLog) (Contact, error)
 
 	// AddURNtoContact adds a URN to the passed in contact
-	AddURNtoContact(context context.Context, channel Channel, contact Contact, urn urns.URN) (urns.URN, error)
+	AddURNtoContact(context context.Context, channel Channel, contact Contact, urn urns.URN, authTokens map[string]string) (urns.URN, error)
 
 	// RemoveURNFromcontact removes a URN from the passed in contact
 	RemoveURNfromContact(context context.Context, channel Channel, contact Contact, urn urns.URN) (urns.URN, error)
 
-	// DeleteMsgWithExternalID delete a message we receive an event that it should be deleted
-	DeleteMsgWithExternalID(ctx context.Context, channel Channel, externalID string) error
+	// DeleteMsgByExternalID deletes a message that has been deleted on the channel side
+	DeleteMsgByExternalID(ctx context.Context, channel Channel, externalID string) error
 
 	// NewIncomingMsg creates a new message from the given params
-	NewIncomingMsg(Channel, urns.URN, string, string, *ChannelLog) Msg
+	NewIncomingMsg(Channel, urns.URN, string, string, *ChannelLog) MsgIn
 
 	// WriteMsg writes the passed in message to our backend
-	WriteMsg(context.Context, Msg, *ChannelLog) error
+	WriteMsg(context.Context, MsgIn, *ChannelLog) error
 
-	// NewMsgStatusForID creates a new Status object for the given message id
-	NewMsgStatusForID(Channel, MsgID, MsgStatusValue, *ChannelLog) MsgStatus
+	// NewStatusUpdate creates a new status update for the given message id
+	NewStatusUpdate(Channel, MsgID, MsgStatus, *ChannelLog) StatusUpdate
 
-	// NewMsgStatusForExternalID creates a new Status object for the given external id
-	NewMsgStatusForExternalID(Channel, string, MsgStatusValue, *ChannelLog) MsgStatus
+	// NewStatusUpdateByExternalID creates a new status update for the given external id
+	NewStatusUpdateByExternalID(Channel, string, MsgStatus, *ChannelLog) StatusUpdate
 
-	// WriteMsgStatus writes the passed in status update to our backend
-	WriteMsgStatus(context.Context, MsgStatus) error
+	// WriteStatusUpdate writes the passed in status update to our backend
+	WriteStatusUpdate(context.Context, StatusUpdate) error
 
 	// NewChannelEvent creates a new channel event for the given channel and event type
 	NewChannelEvent(Channel, ChannelEventType, urns.URN, *ChannelLog) ChannelEvent
@@ -67,7 +69,7 @@ type Backend interface {
 
 	// PopNextOutgoingMsg returns the next message that needs to be sent, callers should call MarkOutgoingMsgComplete with the
 	// returned message when they have dealt with the message (regardless of whether it was sent or not)
-	PopNextOutgoingMsg(context.Context) (Msg, error)
+	PopNextOutgoingMsg(context.Context) (MsgOut, error)
 
 	// WasMsgSent returns whether the backend thinks the passed in message was already sent. This can be used in cases where
 	// a backend wants to implement a failsafe against double sending messages (say if they were double queued)
@@ -80,14 +82,18 @@ type Backend interface {
 	// MarkOutgoingMsgComplete marks the passed in message as having been processed. Note this should be called even in the case
 	// of errors during sending as it will manage the number of active workers per channel. The optional status parameter can be
 	// used to determine any sort of deduping of msg sends
-	MarkOutgoingMsgComplete(context.Context, Msg, MsgStatus)
+	MarkOutgoingMsgComplete(context.Context, MsgOut, StatusUpdate)
 
 	// SaveAttachment saves an attachment to backend storage
 	SaveAttachment(context.Context, Channel, string, []byte, string) (string, error)
 
 	// ResolveMedia resolves an outgoing attachment URL to a media object
 	ResolveMedia(context.Context, string) (Media, error)
 
+	// HttpClient returns an HTTP client for making external requests
+	HttpClient(bool) *http.Client
+	HttpAccess() *httpx.AccessConfig
+
 	// Health returns a string describing any health problems the backend has, or empty string if all is well
 	Health() string