kill-the-newsletter/source/index.ts

528 lines
16 KiB
TypeScript
Raw Normal View History

2021-03-10 22:33:43 +01:00
#!/usr/bin/env node
2021-03-10 11:30:08 +01:00
import path from "path";
2020-05-05 08:12:57 +02:00
import express from "express";
import { SMTPServer } from "smtp-server";
import mailparser from "mailparser";
2021-03-10 11:30:08 +01:00
import fs from "fs-extra";
2020-05-05 08:12:57 +02:00
import cryptoRandomString from "crypto-random-string";
2021-03-10 12:05:04 +01:00
import { html, HTML } from "@leafac/html";
import { css, process as processCSS } from "@leafac/css";
2021-03-13 11:19:52 +01:00
import javascript from "tagged-template-noop";
2021-03-10 12:35:48 +01:00
import { sql, Database } from "@leafac/sqlite";
import databaseMigrate from "@leafac/sqlite-migration";
2020-05-05 08:12:57 +02:00
2021-03-10 11:30:08 +01:00
const VERSION = require("../package.json").version;
2020-05-05 08:12:57 +02:00
2021-03-10 11:30:08 +01:00
export default function killTheNewsletter(
rootDirectory: string
): { webApplication: express.Express; emailApplication: SMTPServer } {
const webApplication = express();
const baseUrl = process.env.BASE_URL ?? "http://localhost:4000";
webApplication.set("url", baseUrl);
const smtpUrl = process.env.SMTP_URL ?? "smtp://localhost:2525";
webApplication.set("email", smtpUrl);
const issueReportEmail =
process.env.ISSUE_REPORT_EMAIL ?? "kill-the-newsletter@leafac.com";
webApplication.set("administrator", `mailto:${issueReportEmail}`);
2021-03-10 12:05:04 +01:00
2021-03-10 22:33:43 +01:00
fs.ensureDirSync(rootDirectory);
2021-03-10 12:35:48 +01:00
const database = new Database(
path.join(rootDirectory, "kill-the-newsletter.db")
);
databaseMigrate(database, [
sql`
CREATE TABLE "feeds" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
2021-03-13 10:28:00 +01:00
"reference" TEXT NOT NULL UNIQUE,
2021-03-10 12:35:48 +01:00
"title" TEXT NOT NULL
);
CREATE TABLE "entries" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
2021-03-13 10:28:00 +01:00
"reference" TEXT NOT NULL UNIQUE,
2021-03-10 12:35:48 +01:00
"feed" INTEGER NOT NULL REFERENCES "feeds",
"title" TEXT NOT NULL,
"author" TEXT NOT NULL,
"content" TEXT NOT NULL
);
`,
2021-03-16 03:18:07 +01:00
sql`
CREATE INDEX "entriesFeed" ON "entries" ("feed");
`,
2021-03-10 12:35:48 +01:00
]);
2021-03-10 12:05:04 +01:00
webApplication.use(express.static(path.join(__dirname, "../public")));
webApplication.use(express.urlencoded({ extended: true }));
const logo = fs.readFileSync(path.join(__dirname, "../public/logo.svg"));
function layout(body: HTML): HTML {
return processCSS(html`
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta
name="viewport"
content="width=device-width, initial-scale=1.0"
/>
2021-03-11 11:08:51 +01:00
<meta name="generator" content="Kill the Newsletter!/${VERSION}" />
<meta
name="description"
content="Convert email newsletters into Atom feeds."
/>
2021-03-10 12:05:04 +01:00
<title>Kill the Newsletter!</title>
</head>
<body
style="${css`
@at-root {
body {
2021-03-18 16:32:40 +01:00
font-size: 0.875rem;
2021-03-18 22:28:32 +01:00
-webkit-text-size-adjust: 100%;
2021-03-18 16:32:40 +01:00
line-height: 1.5;
2021-03-10 12:05:04 +01:00
font-family: --apple-system, BlinkMacSystemFont, "Segoe UI",
Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans",
"Helvetica Neue", sans-serif;
2021-03-11 11:33:19 +01:00
max-width: 450px;
2021-03-18 16:32:40 +01:00
padding: 0 1rem;
margin: 1rem auto;
2021-03-10 12:05:04 +01:00
text-align: center;
2021-03-14 23:00:25 +01:00
overflow-wrap: break-word;
2021-03-18 22:05:53 +01:00
@media (prefers-color-scheme: dark) {
2021-03-22 16:50:24 +01:00
color: #d4d4d4;
2021-03-18 22:05:53 +01:00
background-color: #1e1e1e;
}
2021-03-10 12:05:04 +01:00
}
code {
font-family: SFMono-Regular, Consolas, "Liberation Mono", Menlo,
monospace;
}
h1 {
2021-03-18 16:32:40 +01:00
font-size: 1.3rem;
2021-03-10 12:05:04 +01:00
}
footer {
2021-03-18 16:32:40 +01:00
font-size: 0.75rem;
2021-03-10 12:05:04 +01:00
}
2021-03-18 11:47:46 +01:00
a {
color: inherit;
transition: color 0.2s;
&:hover {
color: #29adff;
}
h1 &,
footer & {
text-decoration: none;
}
}
2021-03-10 12:05:04 +01:00
input,
button {
2021-03-18 10:45:19 +01:00
all: unset;
2021-03-18 22:05:53 +01:00
padding: 0.1rem 1rem;
border: 1px solid gainsboro;
2021-03-18 11:47:46 +01:00
border-radius: 5px;
2021-04-09 15:23:33 +02:00
box-shadow: inset 0 1px 1px #ffffff10, 0 1px 3px #00000010;
2021-03-19 16:33:24 +01:00
transition: border-color 0.2s;
2021-03-18 11:47:46 +01:00
@media (prefers-color-scheme: dark) {
border-color: dimgray;
}
2021-03-10 12:05:04 +01:00
2021-03-23 18:11:54 +01:00
@supports (-webkit-touch-callout: none) {
font-size: 16px;
}
2021-03-10 12:05:04 +01:00
&:focus {
2021-03-18 11:47:46 +01:00
border-color: #29adff;
2021-03-10 12:05:04 +01:00
}
}
2021-03-13 11:19:52 +01:00
button {
2021-03-18 22:05:53 +01:00
background-color: white;
2021-03-13 11:19:52 +01:00
2021-03-18 10:45:19 +01:00
@media (prefers-color-scheme: dark) {
background-color: #5a5a5a;
}
&:active {
color: white;
2021-03-18 11:47:46 +01:00
background-color: #29adff;
2021-03-18 10:45:19 +01:00
}
2021-03-10 12:05:04 +01:00
}
}
`}"
>
<header>
<h1>
<a href="${webApplication.get("url")}/">Kill the Newsletter!</a>
</h1>
<p>Convert email newsletters into Atom feeds</p>
<p
style="${css`
@media (prefers-color-scheme: dark) {
path {
2021-03-22 16:50:24 +01:00
fill: #d4d4d4;
2021-03-10 12:05:04 +01:00
}
}
`}"
>
$${logo}
</p>
</header>
<main>$${body}</main>
<footer>
<p>
2021-03-14 22:52:23 +01:00
By <a href="https://leafac.com">Leandro Facchinetti</a> ·
<a href="https://patreon.com/leafac">Patreon</a> ·
<a href="https://paypal.me/LeandroFacchinetti">PayPal</a> ·
2021-03-10 12:05:04 +01:00
<a href="https://github.com/leafac/kill-the-newsletter.com"
>Source</a
2021-03-14 22:52:23 +01:00
> ·
2021-03-10 12:05:04 +01:00
<a href="${webApplication.get("administrator")}"
2021-03-11 11:33:19 +01:00
>Report an issue</a
2021-03-10 12:05:04 +01:00
>
</p>
</footer>
2021-03-11 12:19:23 +01:00
<script>
for (const copyable of document.querySelectorAll(".copyable"))
copyable.insertAdjacentHTML(
"afterend",
2021-04-12 12:43:09 +02:00
$${"`"}$${html`<br />
<button
type="button"
onclick="${javascript`
(async () => {
await navigator.clipboard.writeText("\${copyable.textContent}");
const originalTextContent = this.textContent;
this.textContent = "Copied";
await new Promise(resolve => window.setTimeout(resolve, 500));
this.textContent = originalTextContent;
})();
`}"
>
Copy
</button>`}$${"`"}
2021-03-11 12:19:23 +01:00
);
</script>
2021-03-10 12:05:04 +01:00
</body>
</html>
`);
}
webApplication.get<{}, HTML, {}, {}, {}>("/", (req, res) => {
res.send(
layout(html`
2021-03-18 22:28:32 +01:00
<form
method="POST"
action="${webApplication.get("url")}"
style="${css`
max-width: 300px;
margin: 0 auto;
input,
button {
box-sizing: border-box;
width: 100%;
}
`}"
>
2021-03-10 12:05:04 +01:00
<p>
<input
type="text"
name="name"
2021-03-10 12:35:48 +01:00
placeholder="Newsletter name…"
2021-03-10 12:05:04 +01:00
maxlength="500"
required
2021-03-17 16:49:43 +01:00
pattern=".*\\S.*"
2021-03-10 12:05:04 +01:00
autocomplete="off"
autofocus
/>
2021-03-18 22:28:32 +01:00
</p>
<p>
2021-03-18 22:05:53 +01:00
<button>Create Inbox</button>
2021-03-10 12:05:04 +01:00
</p>
</form>
`)
);
});
2021-03-10 11:30:08 +01:00
2021-03-11 11:04:09 +01:00
webApplication.post<{}, HTML, { name?: string }, {}, {}>("/", (req, res) => {
if (
2021-03-11 12:25:13 +01:00
typeof req.body.name !== "string" ||
2021-03-11 11:04:09 +01:00
req.body.name.trim() === "" ||
req.body.name.length > 500
)
2021-03-14 23:52:43 +01:00
return res.status(422).send(
2021-03-11 11:04:09 +01:00
layout(
2021-03-13 10:28:00 +01:00
html`
<p>
Error: Missing newsletter name.
<a href="${webApplication.get("url")}/"
><strong>Try again</strong></a
>.
</p>
`
2021-03-11 11:04:09 +01:00
)
);
2021-03-12 11:06:07 +01:00
2021-03-13 10:28:00 +01:00
const feedReference = newReference();
const welcomeTitle = `${req.body.name}” inbox created`;
const welcomeContent = html`
<p>
Sign up for the newsletter with<br />
<code class="copyable"
2021-03-13 11:45:13 +01:00
>${feedReference}@${new URL(webApplication.get("email"))
.hostname}</code
2021-03-13 10:28:00 +01:00
>
</p>
<p>
Subscribe to the Atom feed at<br />
<code class="copyable"
>${webApplication.get("url")}/feeds/${feedReference}.xml</code
>
</p>
<p>
<strong>Dont share these addresses.</strong><br />
They contain an identifier that other people could use to send you spam
and to control your newsletter subscriptions.
</p>
<p><strong>Enjoy your readings!</strong></p>
<p>
<a href="${webApplication.get("url")}/"
><strong>Create another inbox</strong></a
>
</p>
`;
database.executeTransaction(() => {
const feedId = database.run(
sql`INSERT INTO "feeds" ("reference", "title") VALUES (${feedReference}, ${req.body.name})`
).lastInsertRowid;
database.run(
sql`
INSERT INTO "entries" ("reference", "feed", "title", "author", "content")
VALUES (
2021-03-13 11:24:51 +01:00
${newReference()},
2021-03-13 10:28:00 +01:00
${feedId},
${welcomeTitle},
2021-03-15 17:25:42 +01:00
${"Kill the Newsletter!"},
2021-03-13 10:28:00 +01:00
${welcomeContent}
)
`
);
});
2021-03-12 11:06:07 +01:00
2021-03-11 11:04:09 +01:00
res.send(
layout(html`
2021-03-13 10:28:00 +01:00
<p><strong>${welcomeTitle}</strong></p>
$${welcomeContent}
2021-03-11 11:04:09 +01:00
`)
);
});
2021-03-11 22:52:54 +01:00
function renderFeed(feedReference: string): HTML | undefined {
const feed = database.get<{
id: number;
updatedAt: string;
title: string;
}>(
sql`SELECT "id", "updatedAt", "title" FROM "feeds" WHERE "reference" = ${feedReference}`
);
if (feed === undefined) return;
const entries = database.all<{
createdAt: string;
reference: string;
title: string;
author: string;
content: string;
}>(
sql`
SELECT "createdAt", "reference", "title", "author", "content"
FROM "entries"
WHERE "feed" = ${feed.id}
2021-03-14 00:01:59 +01:00
ORDER BY "id" DESC
2021-03-11 22:52:54 +01:00
`
);
return html`
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<link
rel="self"
type="application/atom+xml"
href="${webApplication.get("url")}/feeds/${feedReference}.xml"
/>
<link
rel="alternate"
type="text/html"
href="${webApplication.get("url")}/"
/>
<id>urn:kill-the-newsletter:${feedReference}</id>
<title>${feed.title}</title>
<subtitle
>Kill the Newsletter! Inbox:
2021-03-13 11:45:13 +01:00
${feedReference}@${new URL(webApplication.get("email")).hostname}
2021-03-11 22:52:54 +01:00
${webApplication.get("url")}/feeds/${feedReference}.xml</subtitle
>
<updated>${new Date(feed.updatedAt).toISOString()}</updated>
<author><name>Kill the Newsletter!</name></author>
$${entries.map(
(entry) => html`
<entry>
<id>urn:kill-the-newsletter:${entry.reference}</id>
<title>${entry.title}</title>
<author><name>${entry.author}</name></author>
<updated>${new Date(entry.createdAt).toISOString()}</updated>
<link
rel="alternate"
type="text/html"
href="${webApplication.get(
"url"
)}/alternates/${entry.reference}.html"
/>
<content type="html">${entry.content}</content>
</entry>
`
)}
</feed>
`.trim();
}
2021-03-11 11:23:37 +01:00
webApplication.get<{ feedReference: string }, HTML, {}, {}, {}>(
"/feeds/:feedReference.xml",
(req, res, next) => {
2021-03-11 22:52:54 +01:00
const feed = renderFeed(req.params.feedReference);
2021-03-11 11:23:37 +01:00
if (feed === undefined) return next();
2021-04-17 11:40:15 +02:00
res.type("atom").header("X-Robots-Tag", "noindex").send(feed);
2021-03-11 11:23:37 +01:00
}
);
2021-03-11 11:29:05 +01:00
webApplication.get<{ entryReference: string }, HTML, {}, {}, {}>(
"/alternates/:entryReference.html",
(req, res, next) => {
const entry = database.get<{ content: string }>(
sql`SELECT "content" FROM "entries" WHERE "reference" = ${req.params.entryReference}`
);
if (entry === undefined) return next();
res.header("X-Robots-Tag", "noindex").send(entry.content);
}
);
2021-03-11 11:33:19 +01:00
webApplication.use((req, res) => {
res.send(
layout(html`
<p><strong>404 Not found</strong></p>
<p>
<a href="${webApplication.get("url")}/"
><strong>Create a new inbox</strong></a
>
</p>
`)
);
});
2021-03-11 11:29:05 +01:00
2021-03-11 12:22:54 +01:00
const emailApplication = new SMTPServer({
disabledCommands: ["AUTH", "STARTTLS"],
async onData(stream, session, callback) {
try {
const email = await mailparser.simpleParser(stream);
2021-03-11 22:52:54 +01:00
const from = email.from?.text ?? "";
const subject = email.subject ?? "";
const body =
2021-03-11 12:22:54 +01:00
typeof email.html === "string" ? email.html : email.textAsHtml ?? "";
2021-03-12 11:11:53 +01:00
database.executeTransaction(() => {
for (const address of new Set(
session.envelope.rcptTo.map(
(smtpServerAddress) => smtpServerAddress.address
)
)) {
2021-03-15 00:02:38 +01:00
const addressParts = address.split("@");
if (addressParts.length !== 2) continue;
const [feedReference, hostname] = addressParts;
if (hostname !== new URL(webApplication.get("email")).hostname)
continue;
2021-03-12 11:11:53 +01:00
const feed = database.get<{ id: number }>(
sql`SELECT "id" FROM "feeds" WHERE "reference" = ${feedReference}`
);
if (feed === undefined) continue;
2021-03-11 22:52:54 +01:00
database.run(
2021-03-12 11:10:53 +01:00
sql`
2021-03-13 10:28:00 +01:00
INSERT INTO "entries" ("reference", "feed", "title", "author", "content")
VALUES (
${newReference()},
${feed.id},
${subject},
${from},
${body}
)
2021-03-12 11:10:53 +01:00
`
2021-03-11 12:22:54 +01:00
);
2021-03-13 10:28:00 +01:00
database.run(
sql`UPDATE "feeds" SET "updatedAt" = CURRENT_TIMESTAMP WHERE "id" = ${feed.id}`
);
2021-03-12 11:10:53 +01:00
while (renderFeed(feedReference)!.length > 500_000)
database.run(
2021-03-14 00:01:59 +01:00
sql`DELETE FROM "entries" WHERE "feed" = ${feed.id} ORDER BY "id" ASC LIMIT 1`
2021-03-12 11:10:53 +01:00
);
2021-03-12 11:11:53 +01:00
}
});
2021-03-11 12:22:54 +01:00
callback();
} catch (error) {
console.error(
`Failed to receive message: ${JSON.stringify(session, null, 2)}`
);
console.error(error);
stream.resume();
callback(new Error("Failed to receive message. Please try again."));
}
},
});
2021-03-10 11:30:08 +01:00
2021-03-13 10:28:00 +01:00
function newReference(): string {
return cryptoRandomString({
length: 16,
characters: "abcdefghijklmnopqrstuvwxyz0123456789",
});
}
2021-03-10 11:30:08 +01:00
return { webApplication, emailApplication };
}
if (require.main === module) {
console.log(`Kill the Newsletter!/${VERSION}`);
2021-04-17 11:43:46 +02:00
if (process.argv[2] === undefined) {
2021-03-18 12:20:46 +01:00
const { webApplication, emailApplication } = killTheNewsletter(
path.join(process.cwd(), "data")
);
const webPort =
process.env.WEB_PORT ?? new URL(webApplication.get("url")).port;
webApplication.listen(webPort, () => {
console.log(`Web server started at port ${webPort}`);
2021-03-18 12:20:46 +01:00
});
const emailPort =
process.env.EMAIL_PORT ?? new URL(webApplication.get("email")).port;
emailApplication.listen(emailPort, () => {
console.log(`Email server started at port ${emailPort}`);
2021-03-18 12:20:46 +01:00
});
} else {
2021-04-17 11:43:46 +02:00
const configurationFile = path.resolve(process.argv[2]);
2021-03-18 15:30:22 +01:00
require(configurationFile)(require);
console.log(`Configuration loaded from ${configurationFile}.`);
2021-03-18 12:20:46 +01:00
}
2021-03-10 11:30:08 +01:00
}