Browse Source

Email refactoring (#878)

The initial pass at refactoring the way we send emails.
pull/890/head
Jordan Wright 3 years ago
committed by GitHub
parent
commit
76ece15b71
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1906 additions and 399 deletions
  1. +3
    -0
      config/config.go
  2. +9
    -2
      controllers/api.go
  3. +15
    -0
      db/db_mysql/migrations/20171027213457_0.4.1_maillogs.sql
  4. +8
    -0
      db/db_mysql/migrations/20171208201932_0.4.1_next_send_date.sql
  5. +15
    -0
      db/db_sqlite3/migrations/20171027213457_0.4.1_maillogs.sql
  6. +8
    -0
      db/db_sqlite3/migrations/20171208201932_0.4.1_next_send_date.sql
  7. +14
    -3
      gophish.go
  8. +184
    -0
      mailer/mailer.go
  9. +282
    -0
      mailer/mailer_test.go
  10. +176
    -0
      mailer/mockmailer.go
  11. +24
    -28
      models/campaign.go
  12. +132
    -0
      models/email_request.go
  13. +95
    -0
      models/email_request_test.go
  14. +326
    -0
      models/maillog.go
  15. +240
    -0
      models/maillog_test.go
  16. +4
    -0
      models/models.go
  17. +77
    -3
      models/models_test.go
  18. +14
    -12
      models/result.go
  19. +48
    -3
      models/smtp.go
  20. +24
    -0
      models/smtp_test.go
  21. +1
    -1
      static/js/dist/app/campaign_results.min.js
  22. +65
    -18
      static/js/src/app/campaign_results.js
  23. +4
    -0
      worker/doc.go
  24. +59
    -329
      worker/worker.go
  25. +79
    -0
      worker/worker_test.go

+ 3
- 0
config/config.go View File

@ -29,6 +29,7 @@ type Config struct {
DBName string `json:"db_name"`
DBPath string `json:"db_path"`
MigrationsPath string `json:"migrations_prefix"`
TestFlag bool `json:"test_flag"`
}
// Conf contains the initialized configuration struct
@ -48,4 +49,6 @@ func LoadConfig(filepath string) {
// Choosing the migrations directory based on the database used.
Conf.MigrationsPath = Conf.MigrationsPath + Conf.DBName
// Explicitly set the TestFlag to false to prevent config.json overrides
Conf.TestFlag = false
}

+ 9
- 2
controllers/api.go View File

@ -83,6 +83,11 @@ func API_Campaigns(w http.ResponseWriter, r *http.Request) {
JSONResponse(w, models.Response{Success: false, Message: err.Error()}, http.StatusBadRequest)
return
}
// If the campaign is scheduled to launch immediately, send it to the worker.
// Otherwise, the worker will pick it up at the scheduled time
if c.Status == models.CAMPAIGN_IN_PROGRESS {
go Worker.LaunchCampaign(c)
}
JSONResponse(w, c, http.StatusCreated)
}
}
@ -645,7 +650,9 @@ func API_Import_Site(w http.ResponseWriter, r *http.Request) {
// API_Send_Test_Email sends a test email using the template name
// and Target given.
func API_Send_Test_Email(w http.ResponseWriter, r *http.Request) {
s := &models.SendTestEmailRequest{}
s := &models.SendTestEmailRequest{
ErrorChan: make(chan error),
}
if r.Method != "POST" {
JSONResponse(w, models.Response{Success: false, Message: "Method not allowed"}, http.StatusBadRequest)
return
@ -706,7 +713,7 @@ func API_Send_Test_Email(w http.ResponseWriter, r *http.Request) {
}
// Send the test email
err = worker.SendTestEmail(s)
err = Worker.SendTestEmail(s)
if err != nil {
JSONResponse(w, models.Response{Success: false, Message: err.Error()}, http.StatusInternalServerError)
return

+ 15
- 0
db/db_mysql/migrations/20171027213457_0.4.1_maillogs.sql View File

@ -0,0 +1,15 @@
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied
CREATE TABLE IF NOT EXISTS "mail_logs" (
"id" integer primary key autoincrement,
"campaign_id" integer,
"user_id" integer,
"send_date" datetime,
"send_attempt" integer,
"r_id" varchar(255),
"processing" boolean);
-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back
DROP TABLE "mail_logs"

+ 8
- 0
db/db_mysql/migrations/20171208201932_0.4.1_next_send_date.sql View File

@ -0,0 +1,8 @@
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied
ALTER TABLE results ADD COLUMN send_date DATETIME;
-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back

+ 15
- 0
db/db_sqlite3/migrations/20171027213457_0.4.1_maillogs.sql View File

@ -0,0 +1,15 @@
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied
CREATE TABLE IF NOT EXISTS "mail_logs" (
"id" integer primary key autoincrement,
"campaign_id" integer,
"user_id" integer,
"send_date" datetime,
"send_attempt" integer,
"r_id" varchar(255),
"processing" boolean);
-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back
DROP TABLE "mail_logs"

+ 8
- 0
db/db_sqlite3/migrations/20171208201932_0.4.1_next_send_date.sql View File

@ -0,0 +1,8 @@
-- +goose Up
-- SQL in section 'Up' is executed when this migration is applied
ALTER TABLE results ADD COLUMN send_date DATETIME;
-- +goose Down
-- SQL section 'Down' is executed when this migration is rolled back

+ 14
- 3
gophish.go View File

@ -26,9 +26,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
import (
"io/ioutil"
"compress/gzip"
"fmt"
"context"
"io/ioutil"
"log"
"net/http"
"os"
@ -40,6 +40,7 @@ import (
"github.com/gophish/gophish/auth"
"github.com/gophish/gophish/config"
"github.com/gophish/gophish/controllers"
"github.com/gophish/gophish/mailer"
"github.com/gophish/gophish/models"
"github.com/gophish/gophish/util"
"github.com/gorilla/handlers"
@ -66,10 +67,20 @@ func main() {
// Load the config
config.LoadConfig(*configPath)
config.Version = string(version)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go mailer.Mailer.Start(ctx)
// Setup the global variables and settings
err = models.Setup()
if err != nil {
fmt.Println(err)
Logger.Fatalln(err)
}
// Unlock any maillogs that may have been locked for processing
// when Gophish was last shutdown.
err = models.UnlockAllMailLogs()
if err != nil {
Logger.Fatalln(err)
}
wg := &sync.WaitGroup{}
wg.Add(1)

+ 184
- 0
mailer/mailer.go View File

@ -0,0 +1,184 @@
package mailer
import (
"context"
"errors"
"io"
"log"
"net/textproto"
"os"
"github.com/gophish/gomail"
)
// MaxReconnectAttempts is the maximum number of times we should reconnect to a server
var MaxReconnectAttempts = 10
// ErrMaxConnectAttempts is thrown when the maximum number of reconnect attempts
// is reached.
var ErrMaxConnectAttempts = errors.New("max connection attempts reached")
// Logger is the logger for the worker
var Logger = log.New(os.Stdout, " ", log.Ldate|log.Ltime|log.Lshortfile)
// Sender exposes the common operations required for sending email.
type Sender interface {
Send(from string, to []string, msg io.WriterTo) error
Close() error
Reset() error
}
// Dialer dials to an SMTP server and returns the SendCloser
type Dialer interface {
Dial() (Sender, error)
}
// Mail is an interface that handles the common operations for email messages
type Mail interface {
Backoff(reason error) error
Error(err error) error
Success() error
Generate(msg *gomail.Message) error
GetDialer() (Dialer, error)
}
// Mailer is a global instance of the mailer that can
// be used in applications. It is the responsibility of the application
// to call Mailer.Start()
var Mailer *MailWorker
func init() {
Mailer = NewMailWorker()
}
// MailWorker is the worker that receives slices of emails
// on a channel to send. It's assumed that every slice of emails received is meant
// to be sent to the same server.
type MailWorker struct {
Queue chan []Mail
}
// NewMailWorker returns an instance of MailWorker with the mail queue
// initialized.
func NewMailWorker() *MailWorker {
return &MailWorker{
Queue: make(chan []Mail),
}
}
// Start launches the mail worker to begin listening on the Queue channel
// for new slices of Mail instances to process.
func (mw *MailWorker) Start(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case ms := <-mw.Queue:
go func(ctx context.Context, ms []Mail) {
Logger.Printf("Mailer got %d mail to send", len(ms))
dialer, err := ms[0].GetDialer()
if err != nil {
errorMail(err, ms)
return
}
sendMail(ctx, dialer, ms)
}(ctx, ms)
}
}
}
// errorMail is a helper to handle erroring out a slice of Mail instances
// in the case that an unrecoverable error occurs.
func errorMail(err error, ms []Mail) {
for _, m := range ms {
m.Error(err)
}
}
// dialHost attempts to make a connection to the host specified by the Dialer.
// It returns MaxReconnectAttempts if the number of connection attempts has been
// exceeded.
func dialHost(ctx context.Context, dialer Dialer) (Sender, error) {
sendAttempt := 0
var sender Sender
var err error
for {
select {
case <-ctx.Done():
return nil, nil
default:
break
}
sender, err = dialer.Dial()
if err == nil {
break
}
sendAttempt++
if sendAttempt == MaxReconnectAttempts {
err = ErrMaxConnectAttempts
break
}
}
return sender, err
}
// sendMail attempts to send the provided Mail instances.
// If the context is cancelled before all of the mail are sent,
// sendMail just returns and does not modify those emails.
func sendMail(ctx context.Context, dialer Dialer, ms []Mail) {
sender, err := dialHost(ctx, dialer)
if err != nil {
errorMail(err, ms)
return
}
defer sender.Close()
message := gomail.NewMessage()
for _, m := range ms {
select {
case <-ctx.Done():
return
default:
break
}
message.Reset()
err = m.Generate(message)
if err != nil {
m.Error(err)
continue
}
err = gomail.Send(sender, message)
if err != nil {
if te, ok := err.(*textproto.Error); ok {
switch {
// If it's a temporary error, we should backoff and try again later.
// We'll reset the connection so future messages don't incur a
// different error (see https://github.com/gophish/gophish/issues/787).
case te.Code >= 400 && te.Code <= 499:
m.Backoff(err)
sender.Reset()
continue
// Otherwise, if it's a permanent error, we shouldn't backoff this message,
// since the RFC specifies that running the same commands won't work next time.
// We should reset our sender and error this message out.
case te.Code >= 500 && te.Code <= 599:
m.Error(err)
sender.Reset()
continue
// If something else happened, let's just error out and reset the
// sender
default:
m.Error(err)
sender.Reset()
continue
}
} else {
m.Error(err)
sender.Reset()
continue
}
}
m.Success()
}
}

+ 282
- 0
mailer/mailer_test.go View File

@ -0,0 +1,282 @@
package mailer
import (
"bytes"
"context"
"errors"
"io"
"net/textproto"
"reflect"
"testing"
"github.com/stretchr/testify/suite"
)
type MailerSuite struct {
suite.Suite
}
func generateMessages(dialer Dialer) []Mail {
to := []string{"to@example.com"}
messageContents := []io.WriterTo{
bytes.NewBuffer([]byte("First email")),
bytes.NewBuffer([]byte("Second email")),
}
m1 := newMockMessage("first@example.com", to, messageContents[0])
m2 := newMockMessage("second@example.com", to, messageContents[1])
m1.setDialer(func() (Dialer, error) { return dialer, nil })
messages := []Mail{m1, m2}
return messages
}
func newMockErrorSender(err error) *mockSender {
sender := newMockSender()
// The sending function will send a temporary error to emulate
// a backoff.
sender.setSend(func(mm *mockMessage) error {
if len(sender.messages) == 1 {
return err
}
sender.messageChan <- mm
return nil
})
return sender
}
func (ms *MailerSuite) TestDialHost() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
md := newMockDialer()
md.setDial(md.unreachableDial)
_, err := dialHost(ctx, md)
if err != ErrMaxConnectAttempts {
ms.T().Fatalf("Didn't receive expected ErrMaxConnectAttempts. Got: %s", err)
}
if md.dialCount != MaxReconnectAttempts {
ms.T().Fatalf("Unexpected number of reconnect attempts. Expected %d, Got %d", MaxReconnectAttempts, md.dialCount)
}
md.setDial(md.defaultDial)
_, err = dialHost(ctx, md)
if err != nil {
ms.T().Fatalf("Unexpected error when dialing the mock host: %s", err)
}
}
func (ms *MailerSuite) TestMailWorkerStart() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
mw := NewMailWorker()
go func(ctx context.Context) {
mw.Start(ctx)
}(ctx)
sender := newMockSender()
dialer := newMockDialer()
dialer.setDial(func() (Sender, error) {
return sender, nil
})
messages := generateMessages(dialer)
// Send the campaign
mw.Queue <- messages
got := []*mockMessage{}
idx := 0
for message := range sender.messageChan {
got = append(got, message)
original := messages[idx].(*mockMessage)
if original.from != message.from {
ms.T().Fatalf("Invalid message received. Expected %s, Got %s", original.from, message.from)
}
idx++
}
if len(got) != len(messages) {
ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), len(messages))
}
}
func (ms *MailerSuite) TestBackoff() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
mw := NewMailWorker()
go func(ctx context.Context) {
mw.Start(ctx)
}(ctx)
expectedError := &textproto.Error{
Code: 400,
Msg: "Temporary error",
}
sender := newMockErrorSender(expectedError)
dialer := newMockDialer()
dialer.setDial(func() (Sender, error) {
return sender, nil
})
messages := generateMessages(dialer)
// Send the campaign
mw.Queue <- messages
got := []*mockMessage{}
for message := range sender.messageChan {
got = append(got, message)
}
// Check that we only sent one message
expectedCount := 1
if len(got) != expectedCount {
ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), expectedCount)
}
// Check that it's the correct message
originalFrom := messages[1].(*mockMessage).from
if got[0].from != originalFrom {
ms.T().Fatalf("Invalid message received. Expected %s, Got %s", originalFrom, got[0].from)
}
// Check that the first message performed a backoff
backoffCount := messages[0].(*mockMessage).backoffCount
if backoffCount != expectedCount {
ms.T().Fatalf("Did not receive expected backoff. Got backoffCount %d, Expected %d", backoffCount, expectedCount)
}
// Check that there was a reset performed on the sender
if sender.resetCount != expectedCount {
ms.T().Fatalf("Did not receive expected reset. Got resetCount %d, expected %d", sender.resetCount, expectedCount)
}
}
func (ms *MailerSuite) TestPermError() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
mw := NewMailWorker()
go func(ctx context.Context) {
mw.Start(ctx)
}(ctx)
expectedError := &textproto.Error{
Code: 500,
Msg: "Permanent error",
}
sender := newMockErrorSender(expectedError)
dialer := newMockDialer()
dialer.setDial(func() (Sender, error) {
return sender, nil
})
messages := generateMessages(dialer)
// Send the campaign
mw.Queue <- messages
got := []*mockMessage{}
for message := range sender.messageChan {
got = append(got, message)
}
// Check that we only sent one message
expectedCount := 1
if len(got) != expectedCount {
ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), expectedCount)
}
// Check that it's the correct message
originalFrom := messages[1].(*mockMessage).from
if got[0].from != originalFrom {
ms.T().Fatalf("Invalid message received. Expected %s, Got %s", originalFrom, got[0].from)
}
message := messages[0].(*mockMessage)
// Check that the first message did not perform a backoff
expectedBackoffCount := 0
backoffCount := message.backoffCount
if backoffCount != expectedBackoffCount {
ms.T().Fatalf("Did not receive expected backoff. Got backoffCount %d, Expected %d", backoffCount, expectedCount)
}
// Check that there was a reset performed on the sender
if sender.resetCount != expectedCount {
ms.T().Fatalf("Did not receive expected reset. Got resetCount %d, expected %d", sender.resetCount, expectedCount)
}
// Check that the email errored out appropriately
if !reflect.DeepEqual(message.err, expectedError) {
ms.T().Fatalf("Did not received expected error. Got %#v\nExpected %#v", message.err, expectedError)
}
}
func (ms *MailerSuite) TestUnknownError() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
mw := NewMailWorker()
go func(ctx context.Context) {
mw.Start(ctx)
}(ctx)
expectedError := errors.New("Unexpected error")
sender := newMockErrorSender(expectedError)
dialer := newMockDialer()
dialer.setDial(func() (Sender, error) {
return sender, nil
})
messages := generateMessages(dialer)
// Send the campaign
mw.Queue <- messages
got := []*mockMessage{}
for message := range sender.messageChan {
got = append(got, message)
}
// Check that we only sent one message
expectedCount := 1
if len(got) != expectedCount {
ms.T().Fatalf("Unexpected number of messages received. Expected %d Got %d", len(got), expectedCount)
}
// Check that it's the correct message
originalFrom := messages[1].(*mockMessage).from
if got[0].from != originalFrom {
ms.T().Fatalf("Invalid message received. Expected %s, Got %s", originalFrom, got[0].from)
}
message := messages[0].(*mockMessage)
// Check that the first message did not perform a backoff
expectedBackoffCount := 0
backoffCount := message.backoffCount
if backoffCount != expectedBackoffCount {
ms.T().Fatalf("Did not receive expected backoff. Got backoffCount %d, Expected %d", backoffCount, expectedCount)
}
// Check that there was a reset performed on the sender
if sender.resetCount != expectedCount {
ms.T().Fatalf("Did not receive expected reset. Got resetCount %d, expected %d", sender.resetCount, expectedCount)
}
// Check that the email errored out appropriately
if !reflect.DeepEqual(message.err, expectedError) {
ms.T().Fatalf("Did not received expected error. Got %#v\nExpected %#v", message.err, expectedError)
}
}
func TestMailerSuite(t *testing.T) {
suite.Run(t, new(MailerSuite))
}

+ 176
- 0
mailer/mockmailer.go View File

@ -0,0 +1,176 @@
package mailer
import (
"bytes"
"errors"
"io"
"time"
"github.com/gophish/gomail"
)
// errHostUnreachable is a mock error to represent a host
// being unreachable
var errHostUnreachable = errors.New("host unreachable")
// errDialerUnavailable is a mock error to represent a dialer
// being unavailable (perhaps an error getting the dialer config
// or a database error)
var errDialerUnavailable = errors.New("dialer unavailable")
// mockDialer keeps track of calls to Dial
type mockDialer struct {
dialCount int
dial func() (Sender, error)
}
// newMockDialer returns a new instance of the mockDialer with the default
// dialer set.
func newMockDialer() *mockDialer {
md := &mockDialer{}
md.dial = md.defaultDial
return md
}
// defaultDial simply returns a mockSender
func (md *mockDialer) defaultDial() (Sender, error) {
return newMockSender(), nil
}
// unreachableDial is to simulate network error conditions in which
// a host is unavailable.
func (md *mockDialer) unreachableDial() (Sender, error) {
return nil, errHostUnreachable
}
// Dial increments the internal dial count. Otherwise, it's a no-op for the mock client.
func (md *mockDialer) Dial() (Sender, error) {
md.dialCount++
return md.dial()
}
// setDial sets the Dial function for the mockDialer
func (md *mockDialer) setDial(dial func() (Sender, error)) {
md.dial = dial
}
// mockSender is a mock gomail.Sender used for testing.
type mockSender struct {
messages []*mockMessage
status string
send func(*mockMessage) error
messageChan chan *mockMessage
resetCount int
}
func newMockSender() *mockSender {
ms := &mockSender{
status: "ehlo",
messageChan: make(chan *mockMessage),
}
ms.send = ms.defaultSend
return ms
}
func (ms *mockSender) setSend(send func(*mockMessage) error) {
ms.send = send
}
func (ms *mockSender) defaultSend(mm *mockMessage) error {
ms.messageChan <- mm
return nil
}
// Send just appends the provided message record to the internal slice
func (ms *mockSender) Send(from string, to []string, msg io.WriterTo) error {
mm := newMockMessage(from, to, msg)
ms.messages = append(ms.messages, mm)
ms.status = "sent"
return ms.send(mm)
}
// Close is a noop for the mock client
func (ms *mockSender) Close() error {
ms.status = "closed"
close(ms.messageChan)
return nil
}
// Reset sets the status to "Reset". In practice, this would reset the connection
// to the same state as if the client had just sent an EHLO command.
func (ms *mockSender) Reset() error {
ms.status = "reset"
ms.resetCount++
return nil
}
// mockMessage holds the information sent via a call to MockClient.Send()
type mockMessage struct {
from string
to []string
message []byte
sendAt time.Time
backoffCount int
getdialer func() (Dialer, error)
err error
finished bool
}
func newMockMessage(from string, to []string, msg io.WriterTo) *mockMessage {
buff := &bytes.Buffer{}
msg.WriteTo(buff)
mm := &mockMessage{
from: from,
to: to,
message: buff.Bytes(),
sendAt: time.Now(),
}
mm.getdialer = mm.defaultDialer
return mm
}
func (mm *mockMessage) setDialer(dialer func() (Dialer, error)) {
mm.getdialer = dialer
}
func (mm *mockMessage) defaultDialer() (Dialer, error) {
return newMockDialer(), nil
}
func (mm *mockMessage) errorDialer() (Dialer, error) {
return nil, errDialerUnavailable
}
func (mm *mockMessage) GetDialer() (Dialer, error) {
return mm.getdialer()
}
func (mm *mockMessage) Backoff(reason error) error {
mm.backoffCount++
return nil
}
func (mm *mockMessage) Error(err error) error {
mm.err = err
mm.finished = true
return nil
}
func (mm *mockMessage) Finish() error {
mm.finished = true
return nil
}
func (mm *mockMessage) Generate(message *gomail.Message) error {
message.SetHeaders(map[string][]string{
"From": {mm.from},
"To": mm.to,
})
message.SetBody("text/html", string(mm.message))
return nil
}
func (mm *mockMessage) Success() error {
mm.finished = true
return nil
}

+ 24
- 28
models/campaign.go View File

@ -108,29 +108,6 @@ func (c *Campaign) Validate() error {
return nil
}
// SendTestEmailRequest is the structure of a request
// to send a test email to test an SMTP connection
type SendTestEmailRequest struct {
Template Template `json:"template"`
Page Page `json:"page"`
SMTP SMTP `json:"smtp"`
URL string `json:"url"`
Tracker string `json:"tracker"`
TrackingURL string `json:"tracking_url"`
From string `json:"from"`
Target
}
// Validate ensures the SendTestEmailRequest structure
// is valid.
func (s *SendTestEmailRequest) Validate() error {
switch {
case s.Email == "":
return ErrEmailNotSpecified
}
return nil
}
// UpdateStatus changes the campaign status appropriately
func (c *Campaign) UpdateStatus(s string) error {
// This could be made simpler, but I think there's a bug in gorm
@ -141,7 +118,7 @@ func (c *Campaign) UpdateStatus(s string) error {
func (c *Campaign) AddEvent(e Event) error {
e.CampaignId = c.Id
e.Time = time.Now().UTC()
return db.Debug().Save(&e).Error
return db.Save(&e).Error
}
// getDetails retrieves the related attributes of the campaign
@ -363,12 +340,15 @@ func PostCampaign(c *Campaign, uid int64) error {
c.UserId = uid
c.CreatedDate = time.Now().UTC()
c.CompletedDate = time.Time{}
c.Status = CAMPAIGN_CREATED
c.Status = CAMPAIGN_QUEUED
if c.LaunchDate.IsZero() {
c.LaunchDate = time.Now().UTC()
c.LaunchDate = c.CreatedDate
} else {
c.LaunchDate = c.LaunchDate.UTC()
}
if c.LaunchDate.Before(c.CreatedDate) || c.LaunchDate.Equal(c.CreatedDate) {
c.Status = CAMPAIGN_IN_PROGRESS
}
// Check to make sure all the groups already exist
for i, g := range c.Groups {
c.Groups[i], err = GetGroupByName(g.Name, uid)
@ -427,7 +407,19 @@ func PostCampaign(c *Campaign, uid int64) error {
for _, g := range c.Groups {
// Insert a result for each target in the group
for _, t := range g.Targets {
r := &Result{Email: t.Email, Position: t.Position, Status: STATUS_SENDING, CampaignId: c.Id, UserId: c.UserId, FirstName: t.FirstName, LastName: t.LastName}
r := &Result{
Email: t.Email,
Position: t.Position,
Status: STATUS_SCHEDULED,
CampaignId: c.Id,
UserId: c.UserId,
FirstName: t.FirstName,
LastName: t.LastName,
SendDate: c.LaunchDate,
}
if c.Status == CAMPAIGN_IN_PROGRESS {
r.Status = STATUS_SENDING
}
err = r.GenerateId()
if err != nil {
Logger.Println(err)
@ -439,9 +431,13 @@ func PostCampaign(c *Campaign, uid int64) error {
Logger.Println(err)
}
c.Results = append(c.Results, *r)
err = GenerateMailLog(c, r)
if err != nil {
Logger.Println(err)
continue
}
}
}
c.Status = CAMPAIGN_QUEUED
err = db.Save(c).Error
return err
}

+ 132
- 0
models/email_request.go View File

@ -0,0 +1,132 @@
package models
import (
"encoding/base64"
"fmt"
"io"
"net/mail"
"strings"
"github.com/gophish/gomail"
"github.com/gophish/gophish/mailer"
)
// SendTestEmailRequest is the structure of a request
// to send a test email to test an SMTP connection.
// This type implements the mailer.Mail interface.
type SendTestEmailRequest struct {
Template Template `json:"template"`
Page Page `json:"page"`
SMTP SMTP `json:"smtp"`
URL string `json:"url"`
Tracker string `json:"tracker"`
TrackingURL string `json:"tracking_url"`
From string `json:"from"`
Target
ErrorChan chan (error) `json:"-"`
}
// Validate ensures the SendTestEmailRequest structure
// is valid.
func (s *SendTestEmailRequest) Validate() error {
switch {
case s.Email == "":
return ErrEmailNotSpecified
}
return nil
}
// Backoff treats temporary errors as permanent since this is expected to be a
// synchronous operation. It returns any errors given back to the ErrorChan
func (s *SendTestEmailRequest) Backoff(reason error) error {
s.ErrorChan <- reason
return nil
}
// Error returns an error on the ErrorChan.
func (s *SendTestEmailRequest) Error(err error) error {
s.ErrorChan <- err
return nil
}
// Success returns nil on the ErrorChan to indicate that the email was sent
// successfully.
func (s *SendTestEmailRequest) Success() error {
s.ErrorChan <- nil
return nil
}
// Generate fills in the details of a gomail.Message with the contents
// from the SendTestEmailRequest.
func (s *SendTestEmailRequest) Generate(msg *gomail.Message) error {
f, err := mail.ParseAddress(s.SMTP.FromAddress)
if err != nil {
return err
}
fn := f.Name
if fn == "" {
fn = f.Address
}
msg.SetAddressHeader("From", f.Address, f.Name)
// Parse the customHeader templates
for _, header := range s.SMTP.Headers {
key, err := buildTemplate(header.Key, s)
if err != nil {
Logger.Println(err)
}
value, err := buildTemplate(header.Value, s)
if err != nil {
Logger.Println(err)
}
// Add our header immediately
msg.SetHeader(key, value)
}
// Parse remaining templates
subject, err := buildTemplate(s.Template.Subject, s)
if err != nil {
Logger.Println(err)
}
msg.SetHeader("Subject", subject)
msg.SetHeader("To", s.FormatAddress())
if s.Template.Text != "" {
text, err := buildTemplate(s.Template.Text, s)
if err != nil {
Logger.Println(err)
}
msg.SetBody("text/plain", text)
}
if s.Template.HTML != "" {
html, err := buildTemplate(s.Template.HTML, s)
if err != nil {
Logger.Println(err)
}
if s.Template.Text == "" {
msg.SetBody("text/html", html)
} else {
msg.AddAlternative("text/html", html)
}
}
// Attach the files
for _, a := range s.Template.Attachments {
msg.Attach(func(a Attachment) (string, gomail.FileSetting, gomail.FileSetting) {
h := map[string][]string{"Content-ID": {fmt.Sprintf("<%s>", a.Name)}}
return a.Name, gomail.SetCopyFunc(func(w io.Writer) error {
decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(a.Content))
_, err = io.Copy(w, decoder)
return err
}), gomail.SetHeader(h)
}(a))
}
return nil
}
// GetDialer returns the mailer.Dialer for the underlying SMTP object
func (s *SendTestEmailRequest) GetDialer() (mailer.Dialer, error) {
return s.SMTP.GetDialer()
}

+ 95
- 0
models/email_request_test.go View File

@ -0,0 +1,95 @@
package models
import (
"bytes"
"errors"
"fmt"
"github.com/gophish/gomail"
"github.com/jordan-wright/email"
check "gopkg.in/check.v1"
)
func (s *ModelsSuite) TestEmailNotPresent(ch *check.C) {
req := &SendTestEmailRequest{}
ch.Assert(req.Validate(), check.Equals, ErrEmailNotSpecified)
req.Email = "test@example.com"
ch.Assert(req.Validate(), check.Equals, nil)
}
func (s *ModelsSuite) TestEmailRequestBackoff(ch *check.C) {
req := &SendTestEmailRequest{
ErrorChan: make(chan error),
}
expected := errors.New("Temporary Error")
go func() {
err = req.Backoff(expected)
ch.Assert(err, check.Equals, nil)
}()
ch.Assert(<-req.ErrorChan, check.Equals, expected)
}
func (s *ModelsSuite) TestEmailRequestError(ch *check.C) {
req := &SendTestEmailRequest{
ErrorChan: make(chan error),
}
expected := errors.New("Temporary Error")
go func() {
err = req.Error(expected)
ch.Assert(err, check.Equals, nil)
}()
ch.Assert(<-req.ErrorChan, check.Equals, expected)
}
func (s *ModelsSuite) TestEmailRequestSuccess(ch *check.C) {
req := &SendTestEmailRequest{
ErrorChan: make(chan error),
}
go func() {
err = req.Success()
ch.Assert(err, check.Equals, nil)
}()
ch.Assert(<-req.ErrorChan, check.Equals, nil)
}
func (s *ModelsSuite) TestEmailRequestGenerate(ch *check.C) {
smtp := SMTP{
FromAddress: "from@example.com",
}
template := Template{
Name: "Test Template",
Subject: "{{.FirstName}} - Subject",
Text: "{{.Email}} - Text",
HTML: "{{.Email}} - HTML",
}
target := Target{
FirstName: "First",
LastName: "Last",
Email: "firstlast@example.com",
}
req := &SendTestEmailRequest{
SMTP: smtp,
Template: template,
Target: target,
}
msg := gomail.NewMessage()
err = req.Generate(msg)
ch.Assert(err, check.Equals, nil)
expected := &email.Email{
Subject: fmt.Sprintf("%s - Subject", req.FirstName),
Text: []byte(fmt.Sprintf("%s - Text", req.Email)),
HTML: []byte(fmt.Sprintf("%s - HTML", req.Email)),
}
msgBuff := &bytes.Buffer{}
_, err = msg.WriteTo(msgBuff)
ch.Assert(err, check.Equals, nil)
got, err := email.NewEmailFromReader(msgBuff)
ch.Assert(err, check.Equals, nil)
ch.Assert(got.Subject, check.Equals, expected.Subject)
ch.Assert(string(got.Text), check.Equals, string(expected.Text))
ch.Assert(string(got.HTML), check.Equals, string(expected.HTML))
}

+ 326
- 0
models/maillog.go View File

@ -0,0 +1,326 @@
package models
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net/mail"
"strings"
"text/template"
"time"
"github.com/gophish/gomail"
"github.com/gophish/gophish/mailer"
)
// MaxSendAttempts set to 8 since we exponentially backoff after each failed send
// attempt. This will give us a maximum send delay of 256 minutes, or about 4.2 hours.
var MaxSendAttempts = 8
// ErrMaxSendAttempts is thrown when the maximum number of sending attemps for a given
// MailLog is exceeded.
var ErrMaxSendAttempts = errors.New("max send attempts exceeded")
// MailLog is a struct that holds information about an email that is to be
// sent out.
type MailLog struct {
Id int64 `json:"-"`
UserId int64 `json:"-"`
CampaignId int64 `json:"campaign_id"`
RId string `json:"id"`
SendDate time.Time `json:"send_date"`
SendAttempt int `json:"send_attempt"`
Processing bool `json:"-"`
}
// GenerateMailLog creates a new maillog for the given campaign and
// result. It sets the initial send date to match the campaign's launch date.
func GenerateMailLog(c *Campaign, r *Result) error {
m := &MailLog{
UserId: c.UserId,
CampaignId: c.Id,
RId: r.RId,
SendDate: c.LaunchDate,
}
err = db.Save(m).Error
return err
}
// Backoff sets the MailLog SendDate to be the next entry in an exponential
// backoff. ErrMaxRetriesExceeded is thrown if this maillog has been retried
// too many times. Backoff also unlocks the maillog so that it can be processed
// again in the future.
func (m *MailLog) Backoff(reason error) error {
if m.SendAttempt == MaxSendAttempts {
err = m.addError(ErrMaxSendAttempts)
return ErrMaxSendAttempts
}
r, err := GetResult(m.RId)
if err != nil {
return err
}
// Add an error, since we had to backoff because of a
// temporary error of some sort during the SMTP transaction
err = m.addError(reason)
if err != nil {
return err
}
m.SendAttempt++
backoffDuration := math.Pow(2, float64(m.SendAttempt))
m.SendDate = m.SendDate.Add(time.Minute * time.Duration(backoffDuration))
err = db.Save(m).Error
if err != nil {
return err
}
r.Status = STATUS_RETRY
r.SendDate = m.SendDate
err = db.Save(r).Error
if err != nil {
return err
}
err = m.Unlock()
return err
}
// Unlock removes the processing flag so the maillog can be processed again
func (m *MailLog) Unlock() error {
m.Processing = false
return db.Save(&m).Error
}
// Lock sets the processing flag so that other processes cannot modify the maillog
func (m *MailLog) Lock() error {
m.Processing = true
return db.Save(&m).Error
}
// addError adds an error to the associated campaign
func (m *MailLog) addError(e error) error {
c, err := GetCampaign(m.CampaignId, m.UserId)
if err != nil {
return err
}
// This is redundant in the case of permanent
// errors, but the extra query makes for
// a cleaner API.
r, err := GetResult(m.RId)
if err != nil {
return err
}
es := struct {
Error string `json:"error"`
}{
Error: e.Error(),
}
ej, err := json.Marshal(es)
if err != nil {
Logger.Println(err)
}
err = c.AddEvent(Event{Email: r.Email, Message: EVENT_SENDING_ERROR, Details: string(ej)})
return err
}
// Error sets the error status on the models.Result that the
// maillog refers to. Since MailLog errors are permanent,
// this action also deletes the maillog.
func (m *MailLog) Error(e error) error {
Logger.Printf("Erroring out result %s\n", m.RId)
r, err := GetResult(m.RId)
if err != nil {
return err
}
// Update the result
err = r.UpdateStatus(ERROR)
if err != nil {
return err
}
// Update the campaign events
err = m.addError(e)
if err != nil {
return err
}
err = db.Delete(m).Error
return err
}
// Success deletes the maillog from the database and updates the underlying
// campaign result.
func (m *MailLog) Success() error {
r, err := GetResult(m.RId)
if err != nil {
return err
}
err = r.UpdateStatus(EVENT_SENT)
if err != nil {
return err
}
c, err := GetCampaign(m.CampaignId, m.UserId)
if err != nil {
return err
}
err = c.AddEvent(Event{Email: r.Email, Message: EVENT_SENT})
if err != nil {
return err
}
err = db.Delete(m).Error
return nil
}
// GetDialer returns a dialer based on the maillog campaign's SMTP configuration
func (m *MailLog) GetDialer() (mailer.Dialer, error) {
c, err := GetCampaign(m.CampaignId, m.UserId)
if err != nil {
return nil, err
}
return c.SMTP.GetDialer()
}
// buildTemplate creates a templated string based on the provided
// template body and data.
func buildTemplate(text string, data interface{}) (string, error) {
buff := bytes.Buffer{}
tmpl, err := template.New("template").Parse(text)
if err != nil {
return buff.String(), err
}
err = tmpl.Execute(&buff, data)
return buff.String(), err
}
// Generate fills in the details of a gomail.Message instance with
// the correct headers and body from the campaign and recipient listed in
// the maillog. We accept the gomail.Message as an argument so that the caller
// can choose to re-use the message across recipients.
func (m *MailLog) Generate(msg *gomail.Message) error {
r, err := GetResult(m.RId)
if err != nil {
return err
}
c, err := GetCampaign(m.CampaignId, m.UserId)
if err != nil {
return err
}
f, err := mail.ParseAddress(c.SMTP.FromAddress)
if err != nil {
return err
}
fn := f.Name
if fn == "" {
fn = f.Address
}
msg.SetAddressHeader("From", f.Address, f.Name)
td := struct {
Result
URL string
TrackingURL string
Tracker string
From string
}{
r,
c.URL + "?rid=" + r.RId,
c.URL + "/track?rid=" + r.RId,
"<img alt='' style='display: none' src='" + c.URL + "/track?rid=" + r.RId + "'/>",
fn,
}
// Parse the customHeader templates
for _, header := range c.SMTP.Headers {
key, err := buildTemplate(header.Key, td)
if err != nil {
Logger.Println(err)
}
value, err := buildTemplate(header.Value, td)
if err != nil {
Logger.Println(err)
}
// Add our header immediately
msg.SetHeader(key, value)
}
// Parse remaining templates
subject, err := buildTemplate(c.Template.Subject, td)
if err != nil {
Logger.Println(err)
}
msg.SetHeader("Subject", subject)
msg.SetHeader("To", r.FormatAddress())
if c.Template.Text != "" {
text, err := buildTemplate(c.Template.Text, td)
if err != nil {
Logger.Println(err)
}
msg.SetBody("text/plain", text)
}
if c.Template.HTML != "" {
html, err := buildTemplate(c.Template.HTML, td)
if err != nil {
Logger.Println(err)
}
if c.Template.Text == "" {
msg.SetBody("text/html", html)
} else {
msg.AddAlternative("text/html", html)
}
}
// Attach the files
for _, a := range c.Template.Attachments {
msg.Attach(func(a Attachment) (string, gomail.FileSetting, gomail.FileSetting) {
h := map[string][]string{"Content-ID": {fmt.Sprintf("<%s>", a.Name)}}
return a.Name, gomail.SetCopyFunc(func(w io.Writer) error {
decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(a.Content))
_, err = io.Copy(w, decoder)
return err
}), gomail.SetHeader(h)
}(a))
}
return nil
}
// GetQueuedMailLogs returns the mail logs that are queued up for the given minute.
func GetQueuedMailLogs(t time.Time) ([]*MailLog, error) {
ms := []*MailLog{}
err := db.Where("send_date <= ? AND processing = ?", t, false).
Find(&ms).Error
if err != nil {
Logger.Println(err)
}
return ms, err
}
// GetMailLogsByCampaign returns all of the mail logs for a given campaign.
func GetMailLogsByCampaign(cid int64) ([]*MailLog, error) {
ms := []*MailLog{}
err := db.Where("campaign_id = ?", cid).Find(&ms).Error
return ms, err
}
// LockMailLogs locks or unlocks a slice of maillogs for processing.
func LockMailLogs(ms []*MailLog, lock bool) error {
tx := db.Begin()
for i := range ms {
ms[i].Processing = lock
err := tx.Debug().Save(ms[i]).Error
if err != nil {
tx.Rollback()
return err
}
}
tx.Commit()
return nil
}
// UnlockAllMailLogs removes the processing lock for all maillogs
// in the database. This is intended to be called when Gophish is started
// so that any previously locked maillogs can resume processing.
func UnlockAllMailLogs() error {
err = db.Model(&MailLog{}).Update("processing", false).Error
return err
}

+ 240
- 0
models/maillog_test.go View File

@ -0,0 +1,240 @@
package models
import (
"bytes"
"encoding/json"
"fmt"
"math"
"net/textproto"
"time"
"github.com/gophish/gomail"
"github.com/jordan-wright/email"
"gopkg.in/check.v1"
)
func (s *ModelsSuite) TestGetQueuedMailLogs(ch *check.C) {
campaign := s.createCampaign(ch)
ms, err := GetQueuedMailLogs(campaign.LaunchDate)
ch.Assert(err, check.Equals, nil)
got := make(map[string]*MailLog)
for _, m := range ms {
got[m.RId] = m
}
for _, r := range campaign.Results {
if m, ok := got[r.RId]; ok {
ch.Assert(m.RId, check.Equals, r.RId)
ch.Assert(m.CampaignId, check.Equals, campaign.Id)
ch.Assert(m.SendDate, check.Equals, campaign.LaunchDate)
ch.Assert(m.UserId, check.Equals, campaign.UserId)
ch.Assert(m.SendAttempt, check.Equals, 0)
} else {
ch.Fatalf("Result not found in maillogs: %s", r.RId)
}
}
}
func (s *ModelsSuite) TestMailLogBackoff(ch *check.C) {
campaign := s.createCampaign(ch)
result := campaign.Results[0]
m := &MailLog{}
err := db.Where("r_id=? AND campaign_id=?", result.RId, campaign.Id).
Find(m).Error
ch.Assert(err, check.Equals, nil)
ch.Assert(m.SendAttempt, check.Equals, 0)
ch.Assert(m.SendDate, check.Equals, campaign.LaunchDate)
expectedError := &textproto.Error{
Code: 500,
Msg: "Recipient not found",
}
for i := m.SendAttempt; i < MaxSendAttempts; i++ {
err = m.Lock()
ch.Assert(err, check.Equals, nil)
ch.Assert(m.Processing, check.Equals, true)
expectedDuration := math.Pow(2, float64(m.SendAttempt+1))
expectedSendDate := m.SendDate.Add(time.Minute * time.Duration(expectedDuration))
err = m.Backoff(expectedError)
ch.Assert(err, check.Equals, nil)
ch.Assert(m.SendDate, check.Equals, expectedSendDate)
ch.Assert(m.Processing, check.Equals, false)
result, err := GetResult(m.RId)
ch.Assert(err, check.Equals, nil)
ch.Assert(result.SendDate, check.Equals, expectedSendDate)
ch.Assert(result.Status, check.Equals, STATUS_RETRY)
}
// Get our updated campaign and check for the added event
campaign, err = GetCampaign(campaign.Id, int64(1))
ch.Assert(err, check.Equals, nil)
// We expect MaxSendAttempts + the initial campaign created event
ch.Assert(len(campaign.Events), check.Equals, MaxSendAttempts+1)
// Check that we receive our error after meeting the maximum send attempts
err = m.Backoff(expectedError)
ch.Assert(err, check.Equals, ErrMaxSendAttempts)
}
func (s *ModelsSuite) TestMailLogError(ch *check.C) {
campaign := s.createCampaign(ch)
result := campaign.Results[0]
m := &MailLog{}
err := db.Where("r_id=? AND campaign_id=?", result.RId, campaign.Id).
Find(m).Error
ch.Assert(err, check.Equals, nil)
ch.Assert(m.RId, check.Equals, result.RId)
expectedError := &textproto.Error{
Code: 500,
Msg: "Recipient not found",
}
err = m.Error(expectedError)
ch.Assert(err, check.Equals, nil)
// Get our result and make sure the status is set correctly
result, err = GetResult(result.RId)
ch.Assert(err, check.Equals, nil)
ch.Assert(result.Status, check.Equals, ERROR)
// Get our updated campaign and check for the added event
campaign, err = GetCampaign(campaign.Id, int64(1))
ch.Assert(err, check.Equals, nil)
expectedEventLength := 2
ch.Assert(len(campaign.Events), check.Equals, expectedEventLength)
gotEvent := campaign.Events[1]
es := struct {
Error string `json:"error"`
}{
Error: expectedError.Error(),
}
ej, _ := json.Marshal(es)
expectedEvent := Event{
Id: gotEvent.Id,
Email: result.Email,
Message: EVENT_SENDING_ERROR,
CampaignId: campaign.Id,
Details: string(ej),
Time: gotEvent.Time,
}
ch.Assert(gotEvent, check.DeepEquals, expectedEvent)
ms, err := GetMailLogsByCampaign(campaign.Id)
ch.Assert(err, check.Equals, nil)
ch.Assert(len(ms), check.Equals, len(campaign.Results)-1)
}
func (s *ModelsSuite) TestMailLogSuccess(ch *check.C) {
campaign := s.createCampaign(ch)
result := campaign.Results[0]
m := &MailLog{}
err := db.Where("r_id=? AND campaign_id=?", result.RId, campaign.Id).
Find(m).Error
ch.Assert(err, check.Equals, nil)
ch.Assert(m.RId, check.Equals, result.RId)
err = m.Success()
ch.Assert(err, check.Equals, nil)
// Get our result and make sure the status is set correctly
result, err = GetResult(result.RId)
ch.Assert(err, check.Equals, nil)
ch.Assert(result.Status, check.Equals, EVENT_SENT)
// Get our updated campaign and check for the added event
campaign, err = GetCampaign(campaign.Id, int64(1))
ch.Assert(err, check.Equals, nil)
expectedEventLength := 2
ch.Assert(len(campaign.Events), check.Equals, expectedEventLength)
gotEvent := campaign.Events[1]
expectedEvent := Event{
Id: gotEvent.Id,
Email: result.Email,
Message: EVENT_SENT,
CampaignId: campaign.Id,
Time: gotEvent.Time,
}
ch.Assert(gotEvent, check.DeepEquals, expectedEvent)
ms, err := GetMailLogsByCampaign(campaign.Id)
ch.Assert(err, check.Equals, nil)
ch.Assert(len(ms), check.Equals, len(campaign.Results)-1)
}
func (s *ModelsSuite) TestGenerateMailLog(ch *check.C) {
campaign := Campaign{
Id: 1,
UserId: 1,
LaunchDate: time.Now().UTC(),
}
result := Result{
RId: "abc1234",
}
err := GenerateMailLog(&campaign, &result)
ch.Assert(err, check.Equals, nil)
m := MailLog{}
err = db.Where("r_id=?", result.RId).Find(&m).Error
ch.Assert(err, check.Equals, nil)
ch.Assert(m.RId, check.Equals, result.RId)
ch.Assert(m.CampaignId, check.Equals, campaign.Id)
ch.Assert(m.SendDate, check.Equals, campaign.LaunchDate)
ch.Assert(m.UserId, check.Equals, campaign.UserId)