From 80ae8fab52b38b7d76a79edab3f58b455c79b436 Mon Sep 17 00:00:00 2001 From: Leandro Facchinetti Date: Thu, 23 Jul 2020 16:11:41 +0100 Subject: [PATCH] Fixes & refactorings --- index.ts | 210 +++++++++++++++++++++--------------------- package-lock.json | 51 ++++++++++- package.json | 1 + static/styles.css | 3 +- test.ts | 229 +++++++++++++++++++++++++++------------------- 5 files changed, 289 insertions(+), 205 deletions(-) diff --git a/index.ts b/index.ts index d13c498..d753b7d 100644 --- a/index.ts +++ b/index.ts @@ -1,10 +1,11 @@ import express from "express"; import { SMTPServer } from "smtp-server"; import mailparser from "mailparser"; -import { promises as fs } from "fs"; -import * as entities from "entities"; -import { JSDOM } from "jsdom"; import * as sanitizeXMLString from "sanitize-xml-string"; +import * as entities from "entities"; +import R from "escape-string-regexp"; +import { JSDOM } from "jsdom"; +import { promises as fs } from "fs"; import writeFileAtomic from "write-file-atomic"; import cryptoRandomString from "crypto-random-string"; @@ -18,28 +19,26 @@ export const ISSUE_REPORT = export const webServer = express() .use(express.static("static")) .use(express.urlencoded({ extended: true })) - .get("/", (req, res) => - res.send( - layout(` -
-

- - -

-
- `) - ) - ) + .get("/", (req, res) => res.send(layout(newInbox()))) .post("/", async (req, res, next) => { try { const { name } = req.body; const identifier = createIdentifier(); - await writeFileAtomic(alternatePath(identifier), created(identifier)); - await writeFileAtomic(feedPath(identifier), feed(X(name), identifier)); + await writeFileAtomic(feedPath(identifier), feed(identifier, X(name))); + const renderedCreated = created(identifier); + await addEntryToFeed( + identifier, + entry( + createIdentifier(), + `“${X(name)}” Inbox Created`, + "Kill the Newsletter!", + X(renderedCreated) + ) + ); res.send( layout(`

“${H(name)}” Inbox Created

- ${created(identifier)} + ${renderedCreated} `) ); } catch (error) { @@ -47,14 +46,6 @@ export const webServer = express() next(error); } }) - .get("/alternate", (req, res) => - res.send( - layout(` -

Typically each entry in a feed includes a link
to an online version of the same content,
but the content from the entries in a Kill the Newsletter!
feed come from emails—an online version may not even exist—
so you’re reading this instead.

-

Create an Inbox

- `) - ) - ) .listen(WEB_PORT); export const emailServer = new SMTPServer({ @@ -62,57 +53,30 @@ export const emailServer = new SMTPServer({ async onData(stream, session, callback) { try { const email = await mailparser.simpleParser(stream); - const identifier = createIdentifier(); const content = typeof email.html === "string" ? email.html : email.textAsHtml ?? ""; - await writeFileAtomic(alternatePath(identifier), content); - const newEntry = entry( - X(email.subject ?? ""), - X(email.from?.text ?? ""), - X(content), - identifier - ); for (const { address } of session.envelope.rcptTo) { const match = address.match( - new RegExp(`^(?\\w+)@${EMAIL_DOMAIN}$`) + new RegExp(`^(?\\w+)@${R(EMAIL_DOMAIN)}$`) ); if (match?.groups === undefined) continue; - const path = feedPath(match.groups.identifier); - const xmlText = await fs.readFile(path, "utf8").catch(() => null); - if (xmlText === null) continue; - const xml = new JSDOM(xmlText, { contentType: "text/xml" }); - const document = xml.window.document; - const updated = document.querySelector("feed > updated"); - if (updated === null) - throw new Error(`Can’t find ‘updated’ field in feed at ‘${path}’.`); - updated.textContent = now(); - const firstEntry = document.querySelector("feed > entry:first-of-type"); - if (firstEntry !== null) - firstEntry.insertAdjacentHTML("beforebegin", newEntry); - else - document - .querySelector("feed")! - .insertAdjacentHTML("beforeend", newEntry); - while ( - document.querySelector("feed > entry") !== null && - xml.serialize().length > 500_000 - ) { - const lastEntry = document.querySelector("feed > entry:last-of-type"); - const identifier = removeUrn( - lastEntry!.querySelector("id")!.textContent as string - ); - await fs.unlink(alternatePath(identifier)); - lastEntry!.remove(); - } - await writeFileAtomic( - path, - `${xml.serialize()}` - ); + const identifier = match.groups.identifier.toLowerCase(); + await addEntryToFeed( + identifier, + entry( + createIdentifier(), + X(email.subject ?? ""), + X(email.from?.text ?? ""), + X(content) + ) + ).catch((error) => { + console.error(error); + }); } callback(); } catch (error) { console.error( - `Error receiving email: ${JSON.stringify(session, null, 2)}` + `Failed to receive message: ‘${JSON.stringify(session, null, 2)}’` ); console.error(error); stream.resume(); @@ -121,30 +85,76 @@ export const emailServer = new SMTPServer({ }, }).listen(EMAIL_PORT); +async function addEntryToFeed( + identifier: string, + entry: string +): Promise { + const path = feedPath(identifier); + let text; + try { + text = await fs.readFile(path, "utf8"); + } catch { + return; + } + const feed = new JSDOM(text, { contentType: "text/xml" }); + const document = feed.window.document; + const updated = document.querySelector("feed > updated"); + if (updated === null) throw new Error(`Field ‘updated’ not found: ‘${path}’`); + updated.textContent = now(); + const firstEntry = document.querySelector("feed > entry:first-of-type"); + if (firstEntry === null) + document.querySelector("feed")!.insertAdjacentHTML("beforeend", entry); + else firstEntry.insertAdjacentHTML("beforebegin", entry); + const entryDocument = JSDOM.fragment(entry); + await writeFileAtomic( + alternatePath(getEntryIdentifier(entryDocument)), + entities.decodeXML(entryDocument.querySelector("content")!.textContent!) + ); + while (feed.serialize().length > 500_000) { + const entry = document.querySelector("feed > entry:last-of-type"); + if (entry === null) break; + entry.remove(); + const path = alternatePath(getEntryIdentifier(entry)); + await fs.unlink(path).catch(() => { + console.error(`File not found: ‘${path}’`); + }); + } + await writeFileAtomic( + path, + `${feed.serialize()}` + ); +} + function layout(content: string): string { return ` - - - - Kill the Newsletter! - - - - - - - - -
-

Kill the Newsletter!

-

Convert email newsletters into Atom feeds

-

Convert email newsletters into Atom feeds

-
-
${content}
- - - + + + Kill the Newsletter! + + + + + + +
+

Kill the Newsletter!

+

Convert email newsletters into Atom feeds

+

Convert email newsletters into Atom feeds

+
+
${content}
+ + `; +} + +function newInbox(): string { + return ` +
+

+ + +

+
`; } @@ -160,15 +170,13 @@ function created(identifier: string): string { `.trim(); } -function feed(name: string, identifier: string): string { +function feed(identifier: string, name: string): string { return ` - + ${urn(identifier)} ${name} Kill the Newsletter! Inbox: ${feedEmail( @@ -176,21 +184,15 @@ function feed(name: string, identifier: string): string { )} → ${feedURL(identifier)} ${now()} Kill the Newsletter! - ${entry( - `“${name}” Inbox Created`, - "Kill the Newsletter!", - X(created(identifier)), - identifier - )} `; } function entry( + identifier: string, title: string, author: string, - content: string, - identifier: string + content: string ): string { return ` @@ -213,6 +215,10 @@ function createIdentifier(): string { }); } +function getEntryIdentifier(entry: ParentNode): string { + return entry.querySelector("id")!.textContent!.split(":")[2]; +} + function now(): string { return new Date().toISOString(); } @@ -241,10 +247,6 @@ function urn(identifier: string): string { return `urn:kill-the-newsletter:${identifier}`; } -function removeUrn(identifier: string): string { - return identifier.replace(urn(""), ""); -} - function X(string: string): string { return entities.encodeXML(sanitizeXMLString.sanitize(string)); } diff --git a/package-lock.json b/package-lock.json index 9df80dd..4d4358e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -223,6 +223,14 @@ "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + } } }, "color-convert": { @@ -762,6 +770,13 @@ "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + } } }, "color-convert": { @@ -1714,6 +1729,14 @@ "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + } } }, "color-convert": { @@ -1995,6 +2018,13 @@ "has-ansi": "^2.0.0", "strip-ansi": "^3.0.0", "supports-color": "^2.0.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + } } }, "strip-ansi": { @@ -2556,9 +2586,9 @@ "integrity": "sha1-9EvaEtRbvfnLf4Yu5+SCez3TIlQ=" }, "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" }, "escodegen": { "version": "1.14.1", @@ -6327,6 +6357,13 @@ "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + } } }, "color-convert": { @@ -8182,6 +8219,14 @@ "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", "supports-color": "^5.3.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + } } }, "ci-info": { diff --git a/package.json b/package.json index 1564abc..7180bd9 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "caddy-npm": "^2.1.1", "crypto-random-string": "^3.2.0", "entities": "^2.0.0", + "escape-string-regexp": "^4.0.0", "express": "^4.17.1", "jsdom": "^16.2.2", "mailparser": "^2.7.7", diff --git a/static/styles.css b/static/styles.css index 862d3c3..10b1cc9 100644 --- a/static/styles.css +++ b/static/styles.css @@ -2,8 +2,7 @@ @import "node_modules/typeface-pt-mono/index.css"; body { - font-family: "PT Sans", sans-serif; - line-height: 1.5; + font: 16px/1.5 "PT Sans", sans-serif; text-align: center; padding: 0 1em; margin: 2em 0; diff --git a/test.ts b/test.ts index f83af9f..35d8b71 100644 --- a/test.ts +++ b/test.ts @@ -1,4 +1,4 @@ -import { webServer, emailServer, WEB_PORT, EMAIL_PORT, EMAIL_DOMAIN } from "."; +import { webServer, emailServer, BASE_URL, EMAIL_DOMAIN, EMAIL_PORT } from "."; import nodemailer from "nodemailer"; import axios from "axios"; import qs from "qs"; @@ -6,7 +6,18 @@ import { JSDOM } from "jsdom"; test("create feed", async () => { const identifier = await createFeed(); - expect(await getFeed(identifier)).toMatch("My Feed"); + const feed = await getFeed(identifier); + const entry = feed.querySelector("feed > entry:first-of-type")!; + const alternate = await getAlternate( + entry.querySelector("link")!.getAttribute("href")! + ); + expect(feed.querySelector("feed > title")!.textContent).toBe("My Feed"); + expect(entry.querySelector("title")!.textContent).toBe( + "“My Feed” Inbox Created" + ); + expect(alternate.querySelector("p")!.textContent).toMatch( + "Sign up for the newsletter with" + ); }); describe("receive email", () => { @@ -20,8 +31,8 @@ describe("receive email", () => { html: "

HTML content

", }); const after = await getFeed(identifier); - expect(after.match(/(.*)<\/updated>/)![1]).not.toMatch( - before.match(/(.*)<\/updated>/)![1] + expect(after.querySelector("feed > updated")!.textContent).not.toBe( + before.querySelector("feed > updated")!.textContent ); }); @@ -34,9 +45,16 @@ describe("receive email", () => { html: "

HTML content

", }); const feed = await getFeed(identifier); - expect(feed).toMatch("publisher@example.com"); - expect(feed).toMatch("New Message"); - expect(feed).toMatch("HTML content"); + const entry = feed.querySelector("feed > entry:first-of-type")!; + const alternate = await getAlternate( + entry.querySelector("link")!.getAttribute("href")! + ); + expect(entry.querySelector("author > name")!.textContent).toBe( + "publisher@example.com" + ); + expect(entry.querySelector("title")!.textContent).toBe("New Message"); + expect(entry.querySelector("content")!.textContent).toMatch("HTML content"); + expect(alternate.querySelector("p")!.textContent).toMatch("HTML content"); }); test("text content", async () => { @@ -48,7 +66,12 @@ describe("receive email", () => { text: "TEXT content", }); const feed = await getFeed(identifier); - expect(feed).toMatch("TEXT content"); + const entry = feed.querySelector("feed > entry:first-of-type")!; + const alternate = await getAlternate( + entry.querySelector("link")!.getAttribute("href")! + ); + expect(entry.querySelector("content")!.textContent).toMatch("TEXT content"); + expect(alternate.querySelector("p")!.textContent).toMatch("TEXT content"); }); test("rich text content", async () => { @@ -60,8 +83,13 @@ describe("receive email", () => { text: "TEXT content\n\nhttps://www.leafac.com\n\nMore text", }); const feed = await getFeed(identifier); - expect(feed).toMatch("TEXT content"); - expect(feed).toMatch(`href="https://www.leafac.com"`); + const entry = feed.querySelector("feed > entry:first-of-type")!; + const alternate = await getAlternate( + entry.querySelector("link")!.getAttribute("href")! + ); + expect(alternate.querySelector("a")!.getAttribute("href")).toBe( + "https://www.leafac.com" + ); }); test("invalid XML character in HTML", async () => { @@ -73,7 +101,11 @@ describe("receive email", () => { html: "

Invalid XML character (backspace): |\b|💩

", }); const feed = await getFeed(identifier); - expect(feed).toMatch("Invalid XML character (backspace): ||💩"); + const entry = feed.querySelector("feed > entry:first-of-type")!; + expect(entry.querySelector("content")!.textContent).toMatchInlineSnapshot(` + "

Invalid XML character (backspace): ||💩

+ " + `); }); test("invalid XML character in text", async () => { @@ -85,68 +117,26 @@ describe("receive email", () => { text: "Invalid XML character (backspace): |\b|💩", }); const feed = await getFeed(identifier); - expect(feed).toMatch( - "Invalid XML character (backspace): |&#x8;|&#x1F4A9;" + const entry = feed.querySelector("feed > entry:first-of-type")!; + expect(entry.querySelector("content")!.textContent).toMatchInlineSnapshot( + `"

Invalid XML character (backspace): ||💩

"` ); }); - test("missing content", async () => { + test("missing ‘from’", async () => { const identifier = await createFeed(); await emailClient.sendMail({ - from: "publisher@example.com", to: `${identifier}@${EMAIL_DOMAIN}`, subject: "New Message", - }); - const feed = await getFeed(identifier); - expect(feed).toMatch("New Message"); - }); - - test("missing subject", async () => { - const identifier = await createFeed(); - await emailClient.sendMail({ - from: "publisher@example.com", - to: `${identifier}@${EMAIL_DOMAIN}`, html: "

HTML content

", }); const feed = await getFeed(identifier); - expect(feed).toMatch("HTML content"); + const entry = feed.querySelector("feed > entry:first-of-type")!; + expect(entry.querySelector("author > name")!.textContent).toBe(""); + expect(entry.querySelector("title")!.textContent).toBe("New Message"); }); - test("truncation", async () => { - const identifier = await createFeed(); - for (const repetition of [...new Array(4).keys()]) - await emailClient.sendMail({ - from: "publisher@example.com", - to: `${identifier}@${EMAIL_DOMAIN}`, - subject: "New Message", - text: `REPETITION ${repetition} `.repeat(10_000), - }); - const feed = await getFeed(identifier); - expect(feed).toMatch("REPETITION 3"); - expect(feed).not.toMatch("REPETITION 0"); - }); - - test("too big entry", async () => { - const identifier = await createFeed(); - await emailClient.sendMail({ - from: "publisher@example.com", - to: `${identifier}@${EMAIL_DOMAIN}`, - subject: "New Message", - text: `TOO BIG`.repeat(100_000), - }); - expect(await getFeed(identifier)).not.toMatch(""); - await emailClient.sendMail({ - from: "publisher@example.com", - to: `${identifier}@${EMAIL_DOMAIN}`, - subject: "New Message", - text: `NORMAL SIZE`, - }); - const feed = await getFeed(identifier); - expect(feed).toMatch(""); - expect(feed).toMatch("NORMAL SIZE"); - }); - - test("nonexistent address", async () => { + test("nonexistent ‘to’", async () => { await emailClient.sendMail({ from: "publisher@example.com", to: `nonexistent@${EMAIL_DOMAIN}`, @@ -155,67 +145,114 @@ describe("receive email", () => { }); }); - test("missing from", async () => { + test("missing ‘subject’", async () => { const identifier = await createFeed(); await emailClient.sendMail({ + from: "publisher@example.com", to: `${identifier}@${EMAIL_DOMAIN}`, - subject: "New Message", html: "

HTML content

", }); const feed = await getFeed(identifier); - expect(feed).toMatch("HTML content"); + const entry = feed.querySelector("feed > entry:first-of-type")!; + expect(entry.querySelector("title")!.textContent).toBe(""); + expect(entry.querySelector("author > name")!.textContent).toBe( + "publisher@example.com" + ); }); -}); -describe("alternate", () => { - test("HTML content", async () => { + test("missing ‘content’", async () => { const identifier = await createFeed(); await emailClient.sendMail({ from: "publisher@example.com", to: `${identifier}@${EMAIL_DOMAIN}`, subject: "New Message", - html: "

HTML content

", }); const feed = await getFeed(identifier); - const xml = new JSDOM(feed, { contentType: "text/xml" }); - const document = xml.window.document; - const href = document - .querySelector("feed > entry link")! - .getAttribute("href") as string; - const alternate = await getAlternate(href); - expect(feed).toMatch("publisher@example.com"); - expect(feed).toMatch("New Message"); - expect(feed).toMatch("HTML content"); + const entry = feed.querySelector("feed > entry:first-of-type")!; + expect(entry.querySelector("content")!.textContent!.trim()).toBe(""); + expect(entry.querySelector("title")!.textContent).toBe("New Message"); + }); + + test("truncation", async () => { + const identifier = await createFeed(); + const alternatesURLs = new Array(); + for (const repetition of [...new Array(4).keys()]) { + await emailClient.sendMail({ + from: "publisher@example.com", + to: `${identifier}@${EMAIL_DOMAIN}`, + subject: "New Message", + text: `REPETITION ${repetition} `.repeat(10_000), + }); + const feed = await getFeed(identifier); + const entry = feed.querySelector("feed > entry:first-of-type")!; + alternatesURLs.push(entry.querySelector("link")!.getAttribute("href")!); + } + const feed = await getFeed(identifier); + expect( + feed.querySelector("entry:first-of-type > content")!.textContent + ).toMatch("REPETITION 3"); + expect( + feed.querySelector("entry:last-of-type > content")!.textContent + ).toMatch("REPETITION 1"); + expect((await getAlternate(alternatesURLs[3]!)).textContent).toMatch( + "REPETITION 3" + ); + await expect(getAlternate(alternatesURLs[0]!)).rejects.toThrowError(); + }); + + test("too big entry", async () => { + const identifier = await createFeed(); + await emailClient.sendMail({ + from: "publisher@example.com", + to: `${identifier}@${EMAIL_DOMAIN}`, + subject: "New Message", + text: "TOO BIG".repeat(100_000), + }); + expect((await getFeed(identifier)).querySelector("entry")).toBeNull(); + await emailClient.sendMail({ + from: "publisher@example.com", + to: `${identifier}@${EMAIL_DOMAIN}`, + subject: "New Message", + text: `NORMAL SIZE`, + }); + expect( + (await getFeed(identifier)).querySelector("entry > content")!.textContent + ).toMatchInlineSnapshot(`"

NORMAL SIZE

"`); }); }); +const webClient = axios.create({ + baseURL: BASE_URL, +}); +const emailClient = nodemailer.createTransport( + `smtp://${EMAIL_DOMAIN}:${EMAIL_PORT}` +); afterAll(() => { webServer.close(); emailServer.close(); }); -const webClient = axios.create({ - baseURL: `http://localhost:${WEB_PORT}`, -}); -const emailClient = nodemailer.createTransport( - `smtp://localhost:${EMAIL_PORT}` -); - async function createFeed(): Promise { - return ( - await webClient.post( - "/", - qs.stringify({ - name: "My Feed", - }) - ) - ).data.match(/(\w{20}).xml/)![1]; + return JSDOM.fragment( + ( + await webClient.post( + "/", + qs.stringify({ + name: "My Feed", + }) + ) + ).data + ) + .querySelector("code")! + .textContent!.split("@")[0]; } -async function getFeed(identifier: string): Promise { - return (await webClient.get(`/feeds/${identifier}.xml`)).data; +async function getFeed(identifier: string): Promise { + return new JSDOM((await webClient.get(`/feeds/${identifier}.xml`)).data, { + contentType: "text/xml", + }).window.document; } -async function getAlternate(url: string): Promise { - return (await webClient.get(url)).data; +async function getAlternate(url: string): Promise { + return JSDOM.fragment((await webClient.get(url)).data); }