Add API to re-parse an email (#361)

.vscode/settings.json

@@ -2,7 +2,13 @@
   "files.associations": {
     "*.yml.example": "yaml"
   },
-  "cSpell.words": ["Inlines", "querystrings", "Untrash", "untrashed"],
+  "cSpell.words": [
+    "inlines",
+    "querystrings",
+    "reparse",
+    "untrash",
+    "untrashed"
+  ],
   "go.testEnvVars": {
     "DYNAMODB_TABLE": "test",
     "DYNAMODB_ORIGINAL_INDEX": "OriginalMessageIDIndex"

api/emails/reparse/main.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-lambda-go/events"
+	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/harryzcy/mailbox/internal/api"
+	"github.com/harryzcy/mailbox/internal/email"
+	"github.com/harryzcy/mailbox/internal/env"
+	"github.com/harryzcy/mailbox/internal/util/apiutil"
+)
+
+type reparseClient struct {
+	dynamodbSvc *dynamodb.Client
+	s3Svc       *s3.Client
+}
+
+func (c *reparseClient) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+	return c.s3Svc.GetObject(ctx, params, optFns...)
+}
+
+func (c *reparseClient) UpdateItem(ctx context.Context, params *dynamodb.UpdateItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) {
+	return c.dynamodbSvc.UpdateItem(ctx, params, optFns...)
+}
+
+func handler(ctx context.Context, req events.APIGatewayV2HTTPRequest) (apiutil.Response, error) {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	fmt.Println("request received")
+
+	cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(env.Region))
+	if err != nil {
+		fmt.Printf("unable to load SDK config, %v\n", err)
+		return apiutil.NewErrorResponse(http.StatusInternalServerError, "internal error"), nil
+	}
+
+	messageID := req.PathParameters["messageID"]
+	fmt.Printf("request params: [messagesID] %s\n", messageID)
+	if messageID == "" {
+		return apiutil.NewErrorResponse(http.StatusBadRequest, "bad request: invalid messageID"), nil
+	}
+
+	client := &reparseClient{
+		dynamodbSvc: dynamodb.NewFromConfig(cfg),
+		s3Svc:       s3.NewFromConfig(cfg),
+	}
+
+	err = email.Reparse(ctx, client, messageID)
+	if err != nil {
+		if err == api.ErrTooManyRequests {
+			fmt.Println("too many requests")
+			return apiutil.NewErrorResponse(http.StatusTooManyRequests, "too many requests"), nil
+		}
+
+		fmt.Printf("dynamodb read failed: %v\n", err)
+		return apiutil.NewErrorResponse(http.StatusInternalServerError, "internal error"), nil
+	}
+
+	return apiutil.NewSuccessJSONResponse("{\"status\":\"success\"}"), nil
+}
+
+func main() {
+	lambda.Start(handler)
+}

internal/api/aws.go

@@ -105,3 +105,8 @@ type StoreEmailAPI interface {
 	PutItemAPI
 	TransactWriteItemsAPI
 }
+
+type ReparseEmailAPI interface {
+	storage.S3GetObjectAPI
+	UpdateItemAPI
+}

internal/email/reparse.go

@@ -0,0 +1,54 @@
+package email
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
+	"github.com/harryzcy/mailbox/internal/api"
+	"github.com/harryzcy/mailbox/internal/datasource/storage"
+	"github.com/harryzcy/mailbox/internal/env"
+)
+
+// Reparse re-parse an email from S3 and update the DynamoDB record
+func Reparse(ctx context.Context, client api.ReparseEmailAPI, messageID string) error {
+	item := make(map[string]types.AttributeValue)
+
+	emailResult, err := storage.S3.GetEmail(ctx, client, messageID)
+	if err != nil {
+		return err
+	}
+	item["Text"] = &types.AttributeValueMemberS{Value: emailResult.Text}
+	item["HTML"] = &types.AttributeValueMemberS{Value: emailResult.HTML}
+	item["Attachments"] = emailResult.Attachments.ToAttributeValue()
+	item["Inlines"] = emailResult.Inlines.ToAttributeValue()
+	item["OtherParts"] = emailResult.OtherParts.ToAttributeValue()
+
+	_, err = client.UpdateItem(ctx, &dynamodb.UpdateItemInput{
+		TableName: aws.String(env.TableName),
+		Key: map[string]types.AttributeValue{
+			"MessageID": &types.AttributeValueMemberS{Value: messageID},
+		},
+		UpdateExpression: aws.String("SET Text = :text, HTML = :html, Attachments = :attachments, Inlines = :inlines, OtherParts = :others"),
+		ExpressionAttributeValues: map[string]types.AttributeValue{
+			":text":        &types.AttributeValueMemberS{Value: emailResult.Text},
+			":html":        &types.AttributeValueMemberS{Value: emailResult.HTML},
+			":attachments": emailResult.Attachments.ToAttributeValue(),
+			":inlines":     emailResult.Inlines.ToAttributeValue(),
+			":others":      emailResult.OtherParts.ToAttributeValue(),
+		},
+	})
+	if err != nil {
+		if apiErr := new(types.ProvisionedThroughputExceededException); errors.As(err, &apiErr) {
+			return api.ErrTooManyRequests
+		}
+
+		return err
+	}
+
+	fmt.Println("read method finished successfully")
+	return nil
+}

internal/email/reparse_test.go

@@ -0,0 +1,128 @@
+package email
+
+import (
+	"context"
+	"io"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/harryzcy/mailbox/internal/api"
+	"github.com/stretchr/testify/assert"
+)
+
+type mockReparseEmailAPI struct {
+	mockGetObject  func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
+	mockUpdateItem func(ctx context.Context, params *dynamodb.UpdateItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error)
+}
+
+func (m mockReparseEmailAPI) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+	return m.mockGetObject(ctx, params, optFns...)
+}
+
+func (m mockReparseEmailAPI) UpdateItem(ctx context.Context, params *dynamodb.UpdateItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) {
+	return m.mockUpdateItem(ctx, params, optFns...)
+}
+
+func TestReparse(t *testing.T) {
+	exampleMessageID := "test"
+	raw := `From: [email protected]
+Subject: Example message
+Content-Type: multipart/alternative; boundary=Enmime-100
+
+--Enmime-100
+Content-Type: text/plain
+X-Comment: part1
+
+hello!
+--Enmime-100--`
+	tests := []struct {
+		client      func(t *testing.T) api.ReparseEmailAPI
+		messageID   string
+		expectedErr error
+	}{
+		{
+			client: func(t *testing.T) api.ReparseEmailAPI {
+				return mockReparseEmailAPI{
+					mockGetObject: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+						t.Helper()
+						assert.Equal(t, exampleMessageID, *params.Key)
+						return &s3.GetObjectOutput{
+							Body: io.NopCloser(strings.NewReader(raw)),
+						}, nil
+					},
+					mockUpdateItem: func(ctx context.Context, params *dynamodb.UpdateItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) {
+						text := "hello!"
+						html := ""
+						assert.EqualValues(t, &types.AttributeValueMemberS{Value: exampleMessageID}, (*params).Key["MessageID"])
+						assert.Equal(t, &types.AttributeValueMemberS{Value: text}, (*params).ExpressionAttributeValues[":text"])
+						assert.Equal(t, &types.AttributeValueMemberS{Value: html}, (*params).ExpressionAttributeValues[":html"])
+						assert.Empty(t, (*params).ExpressionAttributeValues[":attachments"].(*types.AttributeValueMemberL).Value)
+						assert.Empty(t, (*params).ExpressionAttributeValues[":inlines"].(*types.AttributeValueMemberL).Value)
+						assert.Empty(t, (*params).ExpressionAttributeValues[":others"].(*types.AttributeValueMemberL).Value)
+
+						return &dynamodb.UpdateItemOutput{}, nil
+					},
+				}
+			},
+			messageID: exampleMessageID,
+		},
+		{
+			client: func(t *testing.T) api.ReparseEmailAPI {
+				return mockReparseEmailAPI{
+					mockGetObject: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+						t.Helper()
+						return &s3.GetObjectOutput{}, api.ErrInvalidInput
+					},
+				}
+			},
+			messageID:   exampleMessageID,
+			expectedErr: api.ErrInvalidInput,
+		},
+		{
+			client: func(t *testing.T) api.ReparseEmailAPI {
+				return mockReparseEmailAPI{
+					mockGetObject: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+						t.Helper()
+						return &s3.GetObjectOutput{
+							Body: io.NopCloser(strings.NewReader(raw)),
+						}, nil
+					},
+					mockUpdateItem: func(ctx context.Context, params *dynamodb.UpdateItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) {
+						return &dynamodb.UpdateItemOutput{}, api.ErrInvalidInput
+					},
+				}
+			},
+			messageID:   exampleMessageID,
+			expectedErr: api.ErrInvalidInput,
+		},
+		{
+			client: func(t *testing.T) api.ReparseEmailAPI {
+				return mockReparseEmailAPI{
+					mockGetObject: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
+						t.Helper()
+						return &s3.GetObjectOutput{
+							Body: io.NopCloser(strings.NewReader(raw)),
+						}, nil
+					},
+					mockUpdateItem: func(ctx context.Context, params *dynamodb.UpdateItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.UpdateItemOutput, error) {
+						return &dynamodb.UpdateItemOutput{}, &types.ProvisionedThroughputExceededException{}
+					},
+				}
+			},
+			messageID:   exampleMessageID,
+			expectedErr: api.ErrTooManyRequests,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			ctx := context.TODO()
+			err := Reparse(ctx, test.client(t), test.messageID)
+			assert.Equal(t, test.expectedErr, err)
+		})
+	}
+}

script/build.sh

@@ -12,7 +12,8 @@ BUILD_VERSION=$(git describe --tags --always)
 ENVIRONMENT="env GOOS=linux GOARCH=amd64 CGO_ENABLED=0"
 
 apiFuncs=(
-  "emails/list" "emails/get" "emails/getRaw" "emails/getContent" "emails/read" "emails/trash" "emails/untrash" "emails/delete" "emails/create" "emails/save" "emails/send"
+  "emails/list" "emails/get" "emails/getRaw" "emails/getContent" "emails/read" "emails/trash" "emails/untrash"
+  "emails/delete" "emails/create" "emails/save" "emails/send" "emails/reparse"
   "threads/get" "threads/trash" "threads/untrash" "threads/delete"
 )
 

serverless.yml.example

@@ -197,6 +197,16 @@ functions:
             type: aws_iam
     package:
       artifact: bin/emails_send.zip
+  emailsReparse:
+    handler: bootstrap
+    events:
+      - httpApi:
+          method: POST
+          path: /emails/{messageID}/reparse
+          authorizer:
+            type: aws_iam
+    package:
+      artifact: bin/emails_reparse.zip
   threadsGet:
     handler: bootstrap
     events: