feat: sender address verification

README.md

@@ -78,9 +78,7 @@ $ docker run -d \
 First of all, you can get most tasks done through the web UI, available at http://localhost:3000.
 
 ### 1. Define a user
-To get started with MailWhale, you need to create a **user** first.
-To do so, you can either let the application initialize a default user by supplying `security.seed_users` in [config.default.yml](config.default.yml)).
-Alternatively, you can also register new users at runtime via API or web UI. `security.allow_signup` needs to be set to `true`. 
+To get started with MailWhale, you need to create a **user** first. To do so, register a new user API or web UI. `security.allow_signup` needs to be set to `true`. 
 
 ### 2. Create an API client
 It is good practice to not authenticate against the API as a user directly. Instead, create an **API client** with limited privileges, that could easily be revoked in the future. A client is identified by a **client ID** and a **client secret** (or token), very similar to what you might already be familiar with from AWS APIs. Usually, such a client corresponds to an individual client application of yours, which wants to access MailWhale's API. 
@@ -163,9 +161,10 @@ You can specify configuration options either via a config file (`config.yml`) or
 |---------------------------|---------------------------|--------------|---------------------------------------------------------------------|
 | `env`                           | `MW_ENV`            | `dev`             | Whether to use development- or production settings |
 | `mail.domain`                   | `MW_MAIL_DOMAIN`    | -                 | Default domain for sending mails |
-| `mail.spf_check`                | `MW_MAIL_SPF_CHECK` | `false`           | Whether to validate sender address domains' SPF records |
+| `mail.verify_senders`           | `MW_VERIFY_SENDERS` | `true`            | Whether to validate sender addresses and their domains' SPF records |
 | `web.listen_v4`                 | `MW_WEB_LISTEN_V4`  | `127.0.0.1:3000`  | IP and port for the web server to listen on |
 | `web.cors_origin`               | -                   | [`http://localhost:5000`] | List of URLs which to accept CORS requests for |
+| `web.public_url`                | `MW_PUBLIC_URL`     | `http://localhost:3000` | The URL under which your MailWhale server is available from the public internet |
 | `smtp.host`                     | `MW_SMTP_HOST`      | -                 | SMTP relay host name or IP |
 | `smtp.port`                     | `MW_SMTP_PORT`      | -                 | SMTP relay port |
 | `smtp.username`                 | `MW_SMTP_USER`      | -                 | SMTP relay authentication user name |
@@ -174,17 +173,16 @@ You can specify configuration options either via a config file (`config.yml`) or
 | `store.path`                    | `MW_STORE_PATH`     | `./data.gob.db`   | Target location of the database file |
 | `security.pepper`               | `MW_SECURITY_PEPPER`| -                 | Pepper to use for hashing user passwords |
 | `security.allow_signup`         | `MW_SECURITY_ALLOW_SIGNUP` | `false`    | Whether to allow the registration of new users |
-| `security.seed_users`           | -                   | -                 | List of users to initially populate the database with (see above) |
 
-### SPF Check
+### Sender verification & SPF Check
 By default, mails are sent using a randomly generated address in the `From` header, which belongs to the domain configured via `mail.domain` (i.e. `[email protected]`). Optionally, custom sender addresses can be configured on a per-API-client basis. However, it is recommended to properly configure [SPF](https://en.wikipedia.org/wiki/Sender_Policy_Framework) on that custom domain and instruct MailWhale to verify that configuration.
 
 **As a user**, you need to configure your domain, which you want to use as part of your senders address (e.g. `example.org` for sending mails from `User Server <[email protected]>`), to publish an SPF record that delegates to the domain under which MailWhale is running (e.g. mailwhale.dev).
 ```
 example.org.  IN  TXT v=spf1 include:mailwhale.dev
 ```
 
-**As a server operator** of a MailWhale instance, you need to enable `mail.spf_check` and set your `mail.domain`. For that domain, you need to configure an SPF record that allows your SMTP relay provider's (e.g. Mailbox.org, GMail, SendGrid, etc.) mail servers to be senders. Refer to your provider's documentation, e.g. [this](https://kb.mailbox.org/display/MBOKBEN/How+to+integrate+external+e-mail+accounts). 
+**As a server operator** of a MailWhale instance, you need to enable `mail.verify_senders` and set your `mail.domain` and `web.public_url`. For that domain, you need to configure an SPF record that allows your SMTP relay provider's (e.g. Mailbox.org, GMail, SendGrid, etc.) mail servers to be senders. Refer to your provider's documentation, e.g. [this](https://kb.mailbox.org/display/MBOKBEN/How+to+integrate+external+e-mail+accounts). 
 
 ## 🚀 Features (planned)
 

config.default.yml

@@ -2,9 +2,10 @@ env: dev                              # Affects log level and a few other things
 
 mail:
   domain: mailwhale.dev               # Your server's domain name
-  spf_check: true
+  verify_senders: true                # Whether to send verification mail when adding new sender addresses
 
 web:
+  public_url: 'http://localhost:3000' # Publicly available URL of your instance, required for callback links via e-mail
   listen_v4: '127.0.0.1:3000'         # Where to make the http server listen
   cors_origins:
     - 'http://localhost:5000'
@@ -21,7 +22,4 @@ store:
 
 security:
   pepper: 'sshhh'                     # Change this!
-  allow_signup: false
-  seed_users:
-    - email: '[email protected]'
-      password: 'admin'               # Change this!
+  allow_signup: true

config/config.go

@@ -25,8 +25,8 @@ type EmailPasswordTuple struct {
 }
 
 type mailConfig struct {
-	Domain   string `yaml:"domain" env:"MW_MAIL_DOMAIN"`
-	SpfCheck bool   `yaml:"spf_check" env:"MW_MAIL_SPF_CHECK"`
+	Domain        string `yaml:"domain" env:"MW_MAIL_DOMAIN"`
+	VerifySenders bool   `yaml:"verify_senders" env:"MW_MAIL_VERIFY_SENDERS"`
 }
 
 type smtpConfig struct {
@@ -40,16 +40,16 @@ type smtpConfig struct {
 type webConfig struct {
 	ListenV4    string   `yaml:"listen_v4" default:"127.0.0.1:3000" env:"MW_WEB_LISTEN_V4"`
 	CorsOrigins []string `yaml:"cors_origins" env:"MW_WEB_CORS_ORIGINS"`
+	PublicUrl   string   `yaml:"public_url" default:"https://mailwhale.dev/" env:"MW_WEB_PUBLIC_URL"`
 }
 
 type storeConfig struct {
 	Path string `default:"data.gob.db" env:"MW_STORE_PATH"`
 }
 
 type securityConfig struct {
-	Pepper      string               `env:"MW_SECURITY_PEPPER"`
-	AllowSignup bool                 `env:"MW_SECURITY_ALLOW_SIGNUP" yaml:"allow_signup"`
-	SeedUsers   []EmailPasswordTuple `yaml:"seed_users"`
+	Pepper      string `env:"MW_SECURITY_PEPPER"`
+	AllowSignup bool   `env:"MW_SECURITY_ALLOW_SIGNUP" yaml:"allow_signup"`
 }
 
 type Config struct {
@@ -91,6 +91,10 @@ func Load() *Config {
 	return Get()
 }
 
+func (c *webConfig) GetPublicUrl() string {
+	return strings.TrimSuffix(c.PublicUrl, "/")
+}
+
 func (c *smtpConfig) ConnStr() string {
 	return fmt.Sprintf("%s:%d", c.Host, c.Port)
 }

main.go

@@ -6,7 +6,6 @@ import (
 	"github.com/gorilla/mux"
 	conf "github.com/muety/mailwhale/config"
 	"github.com/muety/mailwhale/service"
-	"github.com/muety/mailwhale/types"
 	"github.com/muety/mailwhale/web/handlers"
 	"github.com/muety/mailwhale/web/routes/api"
 	"github.com/rs/cors"
@@ -36,8 +35,6 @@ func main() {
 	// Services
 	userService = service.NewUserService()
 
-	initDefaults()
-
 	// Global middlewares
 	recoverMiddleware := ghandlers.RecoveryHandler()
 	loggingMiddleware := handlers.NewLoggingMiddleware(logbuch.Info, []string{})
@@ -98,26 +95,3 @@ func listen(handler http.Handler, config *conf.Config) {
 
 	<-make(chan interface{}, 1)
 }
-
-func initDefaults() {
-	for _, u := range config.Security.SeedUsers {
-		if user, err := userService.GetById(u.Email); err == nil {
-			user.Password = u.Password
-			user, err = userService.Update(user)
-			if err != nil {
-				logbuch.Fatal("failed to update user '%s': %v", u.Email, err)
-			}
-			logbuch.Info("updated user '%s'", user.ID)
-			continue
-		}
-
-		user, err := userService.Create(&types.Signup{
-			Email:    u.Email,
-			Password: u.Password,
-		})
-		if err != nil {
-			logbuch.Fatal("failed to create seed user '%s': %v", u.Email, err)
-		}
-		logbuch.Info("created seed user '%s'", user.ID)
-	}
-}

service/mail.go

@@ -0,0 +1,83 @@
+package service
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/muety/mailwhale/config"
+	"github.com/muety/mailwhale/types"
+	"io/ioutil"
+	"os"
+	"text/template"
+)
+
+// Service with methods for sending system mails, not to be confused with SendService
+
+const (
+	tplPath             = "templates"
+	tplNameVerifySender = "sender_verification"
+)
+
+type MailService struct {
+	config      *config.Config
+	sendService *SendService
+}
+
+func NewMailService() *MailService {
+	return &MailService{
+		config:      config.Get(),
+		sendService: NewSendService(),
+	}
+}
+
+func (s *MailService) SendSenderVerification(user *types.User, sender types.SenderAddress, token string) error {
+	tpl, err := s.loadTemplate(tplNameVerifySender)
+	if err != nil {
+		return err
+	}
+
+	type data struct {
+		UserId        string
+		SenderAddress string
+		VerifyLink    string
+	}
+
+	verifyLink := fmt.Sprintf(
+		"%s/api/user/verify?token=%s",
+		s.config.Web.GetPublicUrl(),
+		token,
+	)
+	payload := &data{
+		UserId:        user.ID,
+		SenderAddress: sender.Raw(),
+		VerifyLink:    verifyLink,
+	}
+
+	var rendered bytes.Buffer
+	if err := tpl.Execute(&rendered, payload); err != nil {
+		return err
+	}
+
+	mail := &types.Mail{
+		From:    types.MailAddress(fmt.Sprintf("MailWhale System <system@%s>", s.config.Mail.Domain)),
+		To:      []types.MailAddress{sender.MailAddress},
+		Subject: "Verify your e-mail address for MailWhale",
+	}
+	mail.WithHTML(rendered.String())
+
+	return s.sendService.Send(mail)
+}
+
+func (s *MailService) loadTemplate(tplName string) (*template.Template, error) {
+	tplFile, err := os.Open(fmt.Sprintf("%s/%s.tpl.html", tplPath, tplName))
+	if err != nil {
+		return nil, err
+	}
+	defer tplFile.Close()
+
+	tplData, err := ioutil.ReadAll(tplFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return template.New(tplName).Parse(string(tplData))
+}

service/user.go

@@ -2,21 +2,30 @@ package service
 
 import (
 	"errors"
+	"fmt"
+	"github.com/emvi/logbuch"
 	conf "github.com/muety/mailwhale/config"
 	"github.com/muety/mailwhale/types"
 	"github.com/muety/mailwhale/util"
 	"github.com/timshannon/bolthold"
+	"strings"
 )
 
 type UserService struct {
-	config *conf.Config
-	store  *bolthold.Store
+	config              *conf.Config
+	store               *bolthold.Store
+	spfService          *SpfService
+	mailService         *MailService
+	verificationService *VerificationService
 }
 
 func NewUserService() *UserService {
 	return &UserService{
-		config: conf.Get(),
-		store:  conf.GetStore(),
+		config:              conf.Get(),
+		store:               conf.GetStore(),
+		spfService:          NewSpfService(),
+		mailService:         NewMailService(),
+		verificationService: NewVerificationService(),
 	}
 }
 
@@ -31,34 +40,90 @@ func (s *UserService) GetAll() (users []*types.User, err error) {
 func (s *UserService) GetById(id string) (*types.User, error) {
 	var user types.User
 	err := s.store.Get(id, &user)
-	return &user, err
+	return user.Sanitize(), err
 }
 
 func (s *UserService) Create(signup *types.Signup) (*types.User, error) {
 	user := &types.User{
 		ID:       signup.Email,
 		Password: util.HashBcrypt(signup.Password, s.config.Security.Pepper),
+		Senders:  []types.SenderAddress{},
 	}
 	if !user.IsValid() {
 		return nil, errors.New("can't create user (empty password or invalid e-mail address)")
 	}
 	if err := s.store.Insert(user.ID, user); err != nil {
 		return nil, err
 	}
-	return user, nil
+	return user.Sanitize(), nil
 }
 
-func (s *UserService) Update(user *types.User) (*types.User, error) {
-	user.Password = util.HashBcrypt(user.Password, s.config.Security.Pepper)
+func (s *UserService) Update(user *types.User, update *types.User) (*types.User, error) {
+	if update.Password != "" {
+		user.Password = util.HashBcrypt(update.Password, s.config.Security.Pepper)
+	}
+
+	newSenders := s.extractNewSenders(user, update)
+	if s.config.Mail.VerifySenders {
+		if err := s.spfCheckSenders(newSenders); err != nil {
+			return nil, err
+		}
+		go s.verifySenders(user, newSenders)
+	}
+	user.Senders = update.Senders
+
 	if !user.IsValid() {
 		return nil, errors.New("user data invalid")
 	}
 	if err := s.store.Update(user.ID, user); err != nil {
 		return nil, err
 	}
-	return user, nil
+	return user.Sanitize(), nil
 }
 
 func (s *UserService) Delete(id string) error {
 	return s.store.Delete(id, &types.User{})
 }
+
+func (s *UserService) extractNewSenders(user *types.User, update *types.User) []types.SenderAddress {
+	newSenders := make([]types.SenderAddress, 0)
+	for _, sender := range update.Senders {
+		if user.HasSender(sender.MailAddress) {
+			continue
+		}
+		newSenders = append(newSenders, sender)
+	}
+	return newSenders
+}
+
+func (s *UserService) spfCheckSenders(senders []types.SenderAddress) error {
+	// TODO: parallelize
+	for _, sender := range senders {
+		senderDomain := strings.Split(sender.MailAddress.Raw(), "@")[1]
+		if err := s.spfService.Validate(senderDomain); err != nil {
+			return errors.New(fmt.Sprintf("failed to verify spf entry for domain '%s'", senderDomain))
+		}
+	}
+	return nil
+}
+
+// generates verification tokens for senders addresses and sends them via mail
+func (s *UserService) verifySenders(user *types.User, senders []types.SenderAddress) error {
+	for _, sender := range senders {
+		verification, err := s.verificationService.Create(types.NewVerification(
+			user,
+			types.VerificationScopeSender,
+			sender.String(),
+		))
+		if err != nil {
+			return err
+		}
+		if err := s.mailService.SendSenderVerification(user, sender, verification.Token); err != nil {
+			logbuch.Error("failed to send sender verification to '%s'", sender.MailAddress.String())
+			return err
+		} else {
+			logbuch.Info("sent sender verification mail for user '%s' to '%s'", user.ID, sender.String())
+		}
+	}
+	return nil
+}