kill-the-newsletter/src/index.ts

519 lines
16 KiB
TypeScript
Raw Normal View History

2021-03-10 22:33:43 +01:00
#!/usr/bin/env node
2021-03-10 11:30:08 +01:00
import path from "path";
2020-05-05 08:12:57 +02:00
import express from "express";
import { SMTPServer } from "smtp-server";
import mailparser from "mailparser";
2021-03-10 11:30:08 +01:00
import fs from "fs-extra";
2020-05-05 08:12:57 +02:00
import cryptoRandomString from "crypto-random-string";
2021-03-10 12:05:04 +01:00
import { html, HTML } from "@leafac/html";
import { css, process as processCSS } from "@leafac/css";
2021-03-13 11:19:52 +01:00
import javascript from "tagged-template-noop";
2021-03-10 12:35:48 +01:00
import { sql, Database } from "@leafac/sqlite";
import databaseMigrate from "@leafac/sqlite-migration";
2020-05-05 08:12:57 +02:00
2021-03-10 11:30:08 +01:00
const VERSION = require("../package.json").version;
2020-05-05 08:12:57 +02:00
2021-03-10 11:30:08 +01:00
export default function killTheNewsletter(
rootDirectory: string
): { webApplication: express.Express; emailApplication: SMTPServer } {
const webApplication = express();
2021-03-14 23:13:17 +01:00
webApplication.set("url", "http://localhost:4000");
2021-03-13 11:45:13 +01:00
webApplication.set("email", "smtp://localhost:2525");
2021-03-10 12:05:04 +01:00
webApplication.set("administrator", "mailto:kill-the-newsletter@leafac.com");
2021-03-10 22:33:43 +01:00
fs.ensureDirSync(rootDirectory);
2021-03-10 12:35:48 +01:00
const database = new Database(
path.join(rootDirectory, "kill-the-newsletter.db")
);
databaseMigrate(database, [
sql`
CREATE TABLE "feeds" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
2021-03-13 10:28:00 +01:00
"reference" TEXT NOT NULL UNIQUE,
2021-03-10 12:35:48 +01:00
"title" TEXT NOT NULL
);
CREATE TABLE "entries" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
2021-03-13 10:28:00 +01:00
"reference" TEXT NOT NULL UNIQUE,
2021-03-10 12:35:48 +01:00
"feed" INTEGER NOT NULL REFERENCES "feeds",
"title" TEXT NOT NULL,
"author" TEXT NOT NULL,
"content" TEXT NOT NULL
);
`,
]);
2021-03-10 12:05:04 +01:00
webApplication.use(express.static(path.join(__dirname, "../public")));
webApplication.use(express.urlencoded({ extended: true }));
const logo = fs.readFileSync(path.join(__dirname, "../public/logo.svg"));
function layout(body: HTML): HTML {
return processCSS(html`
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta
name="viewport"
content="width=device-width, initial-scale=1.0"
/>
2021-03-11 11:08:51 +01:00
<meta name="generator" content="Kill the Newsletter!/${VERSION}" />
<meta
name="description"
content="Convert email newsletters into Atom feeds."
/>
<link
rel="icon"
type="image/png"
sizes="32x32"
href="${webApplication.get("url")}/favicon-32x32.png"
/>
<link
rel="icon"
type="image/png"
sizes="16x16"
href="${webApplication.get("url")}/favicon-16x16.png"
/>
<link
rel="icon"
type="image/x-icon"
href="${webApplication.get("url")}/favicon.ico"
/>
2021-03-10 12:05:04 +01:00
<title>Kill the Newsletter!</title>
</head>
<body
style="${css`
@at-root {
body {
font-size: 14px;
font-family: --apple-system, BlinkMacSystemFont, "Segoe UI",
Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans",
"Helvetica Neue", sans-serif;
line-height: 1.5;
2021-03-14 23:15:53 +01:00
-webkit-text-size-adjust: 100%;
2021-03-11 11:33:19 +01:00
max-width: 450px;
2021-03-10 12:05:04 +01:00
padding: 0 1em;
margin: 1em auto;
text-align: center;
2021-03-14 23:00:25 +01:00
overflow-wrap: break-word;
2021-03-10 12:05:04 +01:00
}
code {
font-family: SFMono-Regular, Consolas, "Liberation Mono", Menlo,
monospace;
}
h1 {
font-size: 1.5em;
}
footer {
2021-03-11 09:49:25 +01:00
font-size: 0.857em;
2021-03-10 12:05:04 +01:00
}
input,
button {
font-family: --apple-system, BlinkMacSystemFont, "Segoe UI",
Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans",
"Helvetica Neue", sans-serif;
font-size: 1em;
line-height: 1.5;
color: inherit;
background-color: transparent;
margin: 0;
outline: none;
}
input {
box-sizing: border-box;
width: 100%;
2021-03-11 09:49:25 +01:00
padding: 0.2em 1em;
border: 1px solid darkgray;
border-radius: 10px;
box-shadow: inset 0px 1px #ffffff22, 0px 1px #00000022;
2021-03-10 12:05:04 +01:00
-webkit-appearance: none;
transition: border-color 0.2s;
&:focus {
border-color: #58a6ff;
}
}
2021-03-13 11:19:52 +01:00
a,
button {
transition: color 0.2s;
&:hover {
color: #58a6ff;
}
}
a {
color: inherit;
text-decoration: none;
}
2021-03-10 12:05:04 +01:00
button {
2021-03-11 09:49:25 +01:00
font-weight: bold;
padding: 0;
border: none;
2021-03-10 12:05:04 +01:00
cursor: pointer;
}
@media (prefers-color-scheme: light) {
body {
color: #000000d4;
}
}
@media (prefers-color-scheme: dark) {
body {
color: #ffffffd4;
background-color: #1e1e1e;
}
}
}
`}"
>
<header>
<h1>
<a href="${webApplication.get("url")}/">Kill the Newsletter!</a>
</h1>
<p>Convert email newsletters into Atom feeds</p>
<p
style="${css`
@media (prefers-color-scheme: dark) {
path {
fill: #ffffffd4;
}
}
`}"
>
$${logo}
</p>
</header>
<main>$${body}</main>
<footer>
<p>
2021-03-14 22:52:23 +01:00
By <a href="https://leafac.com">Leandro Facchinetti</a> ·
<a href="https://patreon.com/leafac">Patreon</a> ·
<a href="https://paypal.me/LeandroFacchinetti">PayPal</a> ·
2021-03-10 12:05:04 +01:00
<a href="https://github.com/leafac/kill-the-newsletter.com"
>Source</a
2021-03-14 22:52:23 +01:00
> ·
2021-03-10 12:05:04 +01:00
<a href="${webApplication.get("administrator")}"
2021-03-11 11:33:19 +01:00
>Report an issue</a
2021-03-10 12:05:04 +01:00
>
</p>
</footer>
2021-03-11 12:19:23 +01:00
<script>
for (const copyable of document.querySelectorAll(".copyable"))
copyable.insertAdjacentHTML(
"afterend",
$${"`"}
$${html`
<br />
<button
type="button"
onclick="${javascript`
(async () => {
await navigator.clipboard.writeText("\${copyable.innerText}");
const originalInnerText = this.innerText;
this.innerText = "Copied";
await new Promise(resolve => window.setTimeout(resolve, 500));
this.innerText = originalInnerText;
})();
`}"
>
Copy
</button>
`}
$${"`"}
);
</script>
2021-03-10 12:05:04 +01:00
</body>
</html>
`);
}
webApplication.get<{}, HTML, {}, {}, {}>("/", (req, res) => {
res.send(
layout(html`
2021-03-13 22:20:52 +01:00
<form method="POST" action="${webApplication.get("url")}/">
2021-03-10 12:05:04 +01:00
<p>
<input
type="text"
name="name"
2021-03-10 12:35:48 +01:00
placeholder="Newsletter name…"
2021-03-10 12:05:04 +01:00
maxlength="500"
required
autocomplete="off"
autofocus
/>
</p>
<p><button>Create inbox</button></p>
</form>
`)
);
});
2021-03-10 11:30:08 +01:00
2021-03-11 11:04:09 +01:00
webApplication.post<{}, HTML, { name?: string }, {}, {}>("/", (req, res) => {
if (
2021-03-11 12:25:13 +01:00
typeof req.body.name !== "string" ||
2021-03-11 11:04:09 +01:00
req.body.name.trim() === "" ||
req.body.name.length > 500
)
return res.status(400).send(
layout(
2021-03-13 10:28:00 +01:00
html`
<p>
Error: Missing newsletter name.
<a href="${webApplication.get("url")}/"
><strong>Try again</strong></a
>.
</p>
`
2021-03-11 11:04:09 +01:00
)
);
2021-03-12 11:06:07 +01:00
2021-03-13 10:28:00 +01:00
const feedReference = newReference();
const welcomeTitle = `${req.body.name}” inbox created`;
const welcomeContent = html`
<p>
Sign up for the newsletter with<br />
<code class="copyable"
2021-03-13 11:45:13 +01:00
>${feedReference}@${new URL(webApplication.get("email"))
.hostname}</code
2021-03-13 10:28:00 +01:00
>
</p>
<p>
Subscribe to the Atom feed at<br />
<code class="copyable"
>${webApplication.get("url")}/feeds/${feedReference}.xml</code
>
</p>
<p>
<strong>Dont share these addresses.</strong><br />
They contain an identifier that other people could use to send you spam
and to control your newsletter subscriptions.
</p>
<p><strong>Enjoy your readings!</strong></p>
<p>
<a href="${webApplication.get("url")}/"
><strong>Create another inbox</strong></a
>
</p>
`;
database.executeTransaction(() => {
const feedId = database.run(
sql`INSERT INTO "feeds" ("reference", "title") VALUES (${feedReference}, ${req.body.name})`
).lastInsertRowid;
database.run(
sql`
INSERT INTO "entries" ("reference", "feed", "title", "author", "content")
VALUES (
2021-03-13 11:24:51 +01:00
${newReference()},
2021-03-13 10:28:00 +01:00
${feedId},
${welcomeTitle},
'Kill the Newsletter!',
${welcomeContent}
)
`
);
});
2021-03-12 11:06:07 +01:00
2021-03-11 11:04:09 +01:00
res.send(
layout(html`
2021-03-13 10:28:00 +01:00
<p><strong>${welcomeTitle}</strong></p>
$${welcomeContent}
2021-03-11 11:04:09 +01:00
`)
);
});
2021-03-11 22:52:54 +01:00
function renderFeed(feedReference: string): HTML | undefined {
const feed = database.get<{
id: number;
updatedAt: string;
title: string;
}>(
sql`SELECT "id", "updatedAt", "title" FROM "feeds" WHERE "reference" = ${feedReference}`
);
if (feed === undefined) return;
const entries = database.all<{
createdAt: string;
reference: string;
title: string;
author: string;
content: string;
}>(
sql`
SELECT "createdAt", "reference", "title", "author", "content"
FROM "entries"
WHERE "feed" = ${feed.id}
2021-03-14 00:01:59 +01:00
ORDER BY "id" DESC
2021-03-11 22:52:54 +01:00
`
);
return html`
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<link
rel="self"
type="application/atom+xml"
href="${webApplication.get("url")}/feeds/${feedReference}.xml"
/>
<link
rel="alternate"
type="text/html"
href="${webApplication.get("url")}/"
/>
<id>urn:kill-the-newsletter:${feedReference}</id>
<title>${feed.title}</title>
<subtitle
>Kill the Newsletter! Inbox:
2021-03-13 11:45:13 +01:00
${feedReference}@${new URL(webApplication.get("email")).hostname}
2021-03-11 22:52:54 +01:00
${webApplication.get("url")}/feeds/${feedReference}.xml</subtitle
>
<updated>${new Date(feed.updatedAt).toISOString()}</updated>
<author><name>Kill the Newsletter!</name></author>
$${entries.map(
(entry) => html`
<entry>
<id>urn:kill-the-newsletter:${entry.reference}</id>
<title>${entry.title}</title>
<author><name>${entry.author}</name></author>
<updated>${new Date(entry.createdAt).toISOString()}</updated>
<link
rel="alternate"
type="text/html"
href="${webApplication.get(
"url"
)}/alternates/${entry.reference}.html"
/>
<content type="html">${entry.content}</content>
</entry>
`
)}
</feed>
`.trim();
}
2021-03-11 11:23:37 +01:00
webApplication.get<{ feedReference: string }, HTML, {}, {}, {}>(
"/feeds/:feedReference.xml",
(req, res, next) => {
2021-03-11 22:52:54 +01:00
const feed = renderFeed(req.params.feedReference);
2021-03-11 11:23:37 +01:00
if (feed === undefined) return next();
res
.contentType("application/atom+xml")
.header("X-Robots-Tag", "noindex")
2021-03-11 22:52:54 +01:00
.send(feed);
2021-03-11 11:23:37 +01:00
}
);
2021-03-11 11:29:05 +01:00
webApplication.get<{ entryReference: string }, HTML, {}, {}, {}>(
"/alternates/:entryReference.html",
(req, res, next) => {
const entry = database.get<{ content: string }>(
sql`SELECT "content" FROM "entries" WHERE "reference" = ${req.params.entryReference}`
);
if (entry === undefined) return next();
res.header("X-Robots-Tag", "noindex").send(entry.content);
}
);
2021-03-11 11:33:19 +01:00
webApplication.use((req, res) => {
res.send(
layout(html`
<p><strong>404 Not found</strong></p>
<p>
<a href="${webApplication.get("url")}/"
><strong>Create a new inbox</strong></a
>
</p>
`)
);
});
2021-03-11 11:29:05 +01:00
2021-03-11 12:22:54 +01:00
const emailApplication = new SMTPServer({
disabledCommands: ["AUTH", "STARTTLS"],
async onData(stream, session, callback) {
try {
2021-03-13 11:45:13 +01:00
const atHost = "@" + new URL(webApplication.get("email")).hostname;
2021-03-11 12:22:54 +01:00
const email = await mailparser.simpleParser(stream);
2021-03-11 22:52:54 +01:00
const from = email.from?.text ?? "";
const subject = email.subject ?? "";
const body =
2021-03-11 12:22:54 +01:00
typeof email.html === "string" ? email.html : email.textAsHtml ?? "";
2021-03-12 11:11:53 +01:00
database.executeTransaction(() => {
for (const address of new Set(
session.envelope.rcptTo.map(
(smtpServerAddress) => smtpServerAddress.address
)
)) {
if (!address.endsWith(atHost)) continue;
const feedReference = address.slice(0, -atHost.length);
const feed = database.get<{ id: number }>(
sql`SELECT "id" FROM "feeds" WHERE "reference" = ${feedReference}`
);
if (feed === undefined) continue;
2021-03-11 22:52:54 +01:00
database.run(
2021-03-12 11:10:53 +01:00
sql`
2021-03-13 10:28:00 +01:00
INSERT INTO "entries" ("reference", "feed", "title", "author", "content")
VALUES (
${newReference()},
${feed.id},
${subject},
${from},
${body}
)
2021-03-12 11:10:53 +01:00
`
2021-03-11 12:22:54 +01:00
);
2021-03-13 10:28:00 +01:00
database.run(
sql`UPDATE "feeds" SET "updatedAt" = CURRENT_TIMESTAMP WHERE "id" = ${feed.id}`
);
2021-03-12 11:10:53 +01:00
while (renderFeed(feedReference)!.length > 500_000)
database.run(
2021-03-14 00:01:59 +01:00
sql`DELETE FROM "entries" WHERE "feed" = ${feed.id} ORDER BY "id" ASC LIMIT 1`
2021-03-12 11:10:53 +01:00
);
2021-03-12 11:11:53 +01:00
}
});
2021-03-11 12:22:54 +01:00
callback();
} catch (error) {
console.error(
`Failed to receive message: ${JSON.stringify(session, null, 2)}`
);
console.error(error);
stream.resume();
callback(new Error("Failed to receive message. Please try again."));
}
},
});
2021-03-10 11:30:08 +01:00
2021-03-13 10:28:00 +01:00
function newReference(): string {
return cryptoRandomString({
length: 16,
characters: "abcdefghijklmnopqrstuvwxyz0123456789",
});
}
2021-03-10 11:30:08 +01:00
return { webApplication, emailApplication };
}
if (require.main === module) {
console.log(`Kill the Newsletter!/${VERSION}`);
const configurationFile = path.resolve(
process.argv[2] ?? path.join(process.cwd(), "configuration.js")
);
require(configurationFile)(require);
2021-03-13 11:19:52 +01:00
console.log(`Configuration loaded from ${configurationFile}.`);
2021-03-10 11:30:08 +01:00
}