Initial commit

Author: eaton

.gitignore

@@ -0,0 +1,15 @@
+**/.DS_Store
+*-debug.log
+*-error.log
+/.idea
+/.tsimp
+/.nyc_output
+/dist
+/lib
+/package-lock.json
+/tmp
+/yarn.lock
+node_modules
+/*.env
+/input
+/output

README.md

@@ -0,0 +1,29 @@
+# MBox Streamer
+
+A streaming, event-emitting mbox file parser. Give it a stream (or a path-like pointer to an mbox file) — it will happily read through, firing off a 'message' event with a ParsedMail payload for each one it encounters.
+
+## Installation
+
+`npm i eaton/mbox-streamer`
+
+## Usage
+
+```typescript
+import { MboxStreamer } from "./streamer.js";
+
+const mbs = new MboxStreamer();
+mbs.on('message', message => {
+  console.log(`${message.date?.toISOString()}: ${message.subject}`);
+});
+
+await mbs.parse("./my-email-backup.mbox");
+```
+
+See the [mailparser](https://github.com/nodemailer/mailparser) project for details on the `ParsedMail` object that's generated.
+
+The [mbox-to-json](https://github.com/d4data-official/mbox-to-json) project served as this one's original inspiration; rather than accumulating parsed messages and returning them at the end of the operation, however, we emit each one and let the listeners figure out what to keep or discard.
+
+## TODOs
+
+- [ ] Allow parsing options to be passed into the `parse()` function; the mailparser project has a few nice convenience flags that can speed up the work if (for example) you don't care about grabbing HTML versions of email bodies or extracting file attachments.
+- [ ] Add a pure Transformer version for folks comfortable piping streams rather than using a wrapper class.

package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@eatonfyi/mbox-streamer",
+  "version": "1.0.0",
+  "description": "Read and parse UNIX Mbox files via stream transforms.",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.cts",
+  "exports": {
+    "require": {
+      "types": "./dist/index.d.cts",
+      "default": "./dist/index.cjs"
+    },
+    "import": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "files": [
+    "/dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "pkgroll",
+    "clean": "shx rm -rf dist",
+    "prepare": "npm run clean && npm run build",
+    "test": "ava"
+  },
+  "ava": {
+    "extensions": {
+      "ts": "module"
+    },
+    "nodeArguments": [
+      "--import=tsimp"
+    ]
+  },
+  "author": "eaton",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/node": "^20.11.30",
+    "ava": "^6.1.2",
+    "pkgroll": "^2.0.2",
+    "shx": "^0.3.4",
+    "tsimp": "^2.0.11",
+    "typescript": "^5.4.3"
+  },
+  "dependencies": {
+    "@types/mailparser": "^3.4.4",
+    "mailparser": "^3.6.9",
+    "tiny-typed-emitter": "^2.1.0"
+  }
+}

src/index.ts

@@ -0,0 +1,2 @@
+export * from './streamer.js';
+export * from './transformer.js';

src/streamer.ts

@@ -0,0 +1,45 @@
+import fs from 'fs';
+import { Readable } from 'stream';
+import { ParsedMail, simpleParser } from 'mailparser';
+import { TypedEmitter } from 'tiny-typed-emitter';
+import { MboxTransformer } from './transformer.js';
+
+interface MboxStreamerEvents {
+  message: (message: ParsedMail) => void;
+  error: (err: any) => void;
+  close: () => void;
+}
+
+export class MboxStreamer extends TypedEmitter<MboxStreamerEvents> {
+  constructor() {
+    super();
+  }
+
+  async parse(input: fs.PathLike | Readable, rejectOnError = false) {
+    const stream = input instanceof Readable ? input : fs.createReadStream(input);
+
+    return new Promise<undefined>((resolve, reject) => {
+      const transform = new MboxTransformer();
+      const mbox = stream.pipe(transform);
+
+      mbox.on('data', message => {
+        simpleParser(message, { keepCidLinks: true }, (err, mail) => {
+          if (err) {
+            if (rejectOnError) {
+              reject(err);
+            } else {
+              this.emit('error', err);
+            }
+          } else {
+            this.emit('message', mail);
+          }
+        });
+      });
+
+      mbox.on('close', () => {
+        this.emit('close');
+        resolve(undefined);
+      });
+    });
+  }
+}

src/transformer.ts

@@ -0,0 +1,38 @@
+import { Transform, TransformCallback } from "stream";
+
+export class MboxTransformer extends Transform {
+  private remaining: string;
+
+  constructor() {
+    super({
+      readableObjectMode: true,
+      writableObjectMode: false,
+      encoding: 'binary',
+    });
+    this.remaining = '';
+  }
+
+  _transform(chunk: Buffer, encoding: string, callback: TransformCallback) {
+    const data = `${this.remaining}${chunk.toString()}`;
+    this.remaining = '';
+    const mails = data.split(/^From /m).filter((mail) => mail.length).map((mail) => {
+      return `From ${mail}`
+    });
+    this.remaining = mails.pop() ?? '';
+    if (mails.length > 0) {
+      for (const mail of mails) {
+        this.emit('data', mail);
+      }
+    }
+    callback();
+  };
+
+  _flush(callback: TransformCallback) {
+    if (this.remaining.length) {
+      this.emit('data', this.remaining)
+      this.remaining = '';
+      callback();
+      this.emit('close')
+    }
+  }
+}

tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "module": "ESNext",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "target": "ESNext",
+    "lib": ["ESNext"],
+    "moduleResolution": "Bundler",
+    "skipLibCheck": true,
+    "typeRoots": ["./types", "./node_modules/@types"]
+  },
+  "include": ["./src/**/*"]
+}