Bhasher/kill-the-newsletter
Bhasher
/
kill-the-newsletter
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
kill-the-newsletter/src/index.ts

483 lines
15 KiB
TypeScript
Raw Normal View History

2021-03-10 22:33:43 +01:00
#!/usr/bin/env node
2021-03-10 11:30:08 +01:00
import path from "path";
Change some dependencies
2020-05-05 08:12:57 +02:00
import express from "express";
import { SMTPServer } from "smtp-server";
import mailparser from "mailparser";
2021-03-10 11:30:08 +01:00
import escapeStringRegexp from "escape-string-regexp";
import fs from "fs-extra";
Change some dependencies
2020-05-05 08:12:57 +02:00
import cryptoRandomString from "crypto-random-string";
2021-03-10 12:05:04 +01:00
import { html, HTML } from "@leafac/html";
import { css, process as processCSS } from "@leafac/css";
2021-03-10 12:35:48 +01:00
import { sql, Database } from "@leafac/sqlite";
import databaseMigrate from "@leafac/sqlite-migration";
Change some dependencies
2020-05-05 08:12:57 +02:00
2021-03-10 11:30:08 +01:00
const VERSION = require("../package.json").version;
Change some dependencies
2020-05-05 08:12:57 +02:00
2021-03-10 11:30:08 +01:00
export default function killTheNewsletter(
rootDirectory: string
): { webApplication: express.Express; emailApplication: SMTPServer } {
const webApplication = express();
webApplication.set("url", "http://localhost:4000");
webApplication.set("email port", 2525);
webApplication.set("email host", "localhost");
2021-03-10 12:05:04 +01:00
webApplication.set("administrator", "mailto:kill-the-newsletter@leafac.com");
2021-03-10 22:33:43 +01:00
fs.ensureDirSync(rootDirectory);
2021-03-10 12:35:48 +01:00
const database = new Database(
path.join(rootDirectory, "kill-the-newsletter.db")
);
databaseMigrate(database, [
sql`
CREATE TABLE "feeds" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"reference" TEXT NOT NULL UNIQUE,
"title" TEXT NOT NULL
);
CREATE TABLE "entries" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"reference" TEXT NOT NULL UNIQUE,
"feed" INTEGER NOT NULL REFERENCES "feeds",
"title" TEXT NOT NULL,
"author" TEXT NOT NULL,
"content" TEXT NOT NULL
);
`,
]);
2021-03-10 12:05:04 +01:00
webApplication.use(express.static(path.join(__dirname, "../public")));
webApplication.use(express.urlencoded({ extended: true }));
const logo = fs.readFileSync(path.join(__dirname, "../public/logo.svg"));
function layout(body: HTML): HTML {
return processCSS(html`
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta
name="viewport"
content="width=device-width, initial-scale=1.0"
/>
2021-03-11 11:08:51 +01:00
<meta name="generator" content="Kill the Newsletter!/${VERSION}" />
<meta
name="description"
content="Convert email newsletters into Atom feeds."
/>
<link
rel="icon"
type="image/png"
sizes="32x32"
href="${webApplication.get("url")}/favicon-32x32.png"
/>
<link
rel="icon"
type="image/png"
sizes="16x16"
href="${webApplication.get("url")}/favicon-16x16.png"
/>
<link
rel="icon"
type="image/x-icon"
href="${webApplication.get("url")}/favicon.ico"
/>
2021-03-10 12:05:04 +01:00
<title>Kill the Newsletter!</title>
</head>
<body
style="${css`
@at-root {
body {
font-size: 14px;
font-family: --apple-system, BlinkMacSystemFont, "Segoe UI",
Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans",
"Helvetica Neue", sans-serif;
line-height: 1.5;
max-width: 400px;
padding: 0 1em;
margin: 1em auto;
text-align: center;
}
code {
font-family: SFMono-Regular, Consolas, "Liberation Mono", Menlo,
monospace;
}
2021-03-11 09:49:25 +01:00
a,
button {
2021-03-10 12:05:04 +01:00
color: inherit;
text-decoration: none;
transition: color 0.2s;
&:hover {
color: #58a6ff;
}
}
h1 {
font-size: 1.5em;
}
footer {
2021-03-11 09:49:25 +01:00
font-size: 0.857em;
2021-03-10 12:05:04 +01:00
}
input,
button {
font-family: --apple-system, BlinkMacSystemFont, "Segoe UI",
Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans",
"Helvetica Neue", sans-serif;
font-size: 1em;
line-height: 1.5;
color: inherit;
background-color: transparent;
margin: 0;
outline: none;
}
input {
box-sizing: border-box;
width: 100%;
2021-03-11 09:49:25 +01:00
padding: 0.2em 1em;
border: 1px solid darkgray;
border-radius: 10px;
box-shadow: inset 0px 1px #ffffff22, 0px 1px #00000022;
2021-03-10 12:05:04 +01:00
-webkit-appearance: none;
transition: border-color 0.2s;
&:focus {
border-color: #58a6ff;
}
}
button {
2021-03-11 09:49:25 +01:00
font-weight: bold;
padding: 0;
border: none;
2021-03-10 12:05:04 +01:00
cursor: pointer;
}
@media (prefers-color-scheme: light) {
body {
color: #000000d4;
}
}
@media (prefers-color-scheme: dark) {
body {
color: #ffffffd4;
background-color: #1e1e1e;
}
}
}
`}"
>
<header>
<h1>
<a href="${webApplication.get("url")}/">Kill the Newsletter!</a>
</h1>
<p>Convert email newsletters into Atom feeds</p>
<p
style="${css`
@media (prefers-color-scheme: dark) {
path {
fill: #ffffffd4;
}
}
`}"
>
$${logo}
</p>
</header>
<main>$${body}</main>
<footer>
<p>
By <a href="https://leafac.com">Leandro Facchinetti</a> ·
<a href="https://github.com/leafac/kill-the-newsletter.com"
>Source</a
>
·
<a href="${webApplication.get("administrator")}"
>Report an Issue</a
>
</p>
</footer>
</body>
</html>
`);
}
webApplication.get<{}, HTML, {}, {}, {}>("/", (req, res) => {
res.send(
layout(html`
2021-03-11 11:04:09 +01:00
<form method="post" action="${webApplication.get("url")}/">
2021-03-10 12:05:04 +01:00
<p>
<input
type="text"
name="name"
2021-03-10 12:35:48 +01:00
placeholder="Newsletter name…"
2021-03-10 12:05:04 +01:00
maxlength="500"
required
autocomplete="off"
autofocus
/>
</p>
<p><button>Create inbox</button></p>
</form>
`)
);
});
2021-03-10 11:30:08 +01:00
2021-03-11 11:04:09 +01:00
webApplication.post<{}, HTML, { name?: string }, {}, {}>("/", (req, res) => {
if (
req.body.name === undefined ||
req.body.name.trim() === "" ||
req.body.name.length > 500
)
return res.status(400).send(
layout(
html`<p>
Error: Missing newsletter name.
<a href="${webApplication.get("url")}/"
><strong>Try again</strong></a
>.
</p>`
)
);
const reference = newReference();
const created = html`
<p>
Sign up for the newsletter with<br />
<code class="copyable"
>${reference}@${webApplication.get("email host")}</code
>
</p>
<p>
Subscribe to the Atom feed at<br />
<code class="copyable"
>${webApplication.get("url")}/feeds/${reference}.xml</code
>
</p>
<p>
<strong>Don’t share these addresses.</strong><br />
They contain an identifier that other people could use to send you spam
and to control your newsletter subscriptions.
</p>
<p>Enjoy your readings!</p>
<p>
<a href="${webApplication.get("url")}/"
><strong>Create Another Inbox</strong></a
>
</p>
`;
const feedId = database.run(
sql`INSERT INTO "feeds" ("reference", "title") VALUES (${reference}, ${req.body.name})`
).lastInsertRowid;
database.run(
sql`
INSERT INTO "entries" ("reference", "feed", "title", "author", "content")
VALUES (${newReference()}, ${feedId}, ${`“${req.body.name}” inbox created`}, ${"Kill the Newsletter!"}, ${created})
`
);
res.send(
layout(html`
<p>
<strong>“${req.body.name}” inbox created</strong>
$${created}
</p>
`)
);
});
2021-03-11 11:23:37 +01:00
webApplication.get<{ feedReference: string }, HTML, {}, {}, {}>(
"/feeds/:feedReference.xml",
(req, res, next) => {
const feed = database.get<{
id: number;
updatedAt: string;
title: string;
}>(
sql`SELECT "id", "updatedAt", "title" FROM "feeds" WHERE "reference" = ${req.params.feedReference}`
);
if (feed === undefined) return next();
const entries = database.all<{
createdAt: string;
reference: string;
title: string;
author: string;
content: string;
}>(
sql`
SELECT "createdAt", "reference", "title", "author", "content"
FROM "entries"
WHERE "feed" = ${feed.id}
`
);
res
.contentType("application/atom+xml")
.header("X-Robots-Tag", "noindex")
.send(
html`
<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<link
rel="self"
type="application/atom+xml"
href="${webApplication.get("url")}/feeds/${req.params
.feedReference}.xml"
/>
<link
rel="alternate"
type="text/html"
href="${webApplication.get("url")}/"
/>
<id>urn:kill-the-newsletter:${req.params.feedReference}</id>
<title>${feed.title}</title>
<subtitle
>Kill the Newsletter! Inbox:
${req.params.feedReference}@${webApplication.get("email host")}
→
${webApplication.get("url")}/feeds/${req.params
.feedReference}.xml</subtitle
>
<updated>${new Date(feed.updatedAt).toISOString()}</updated>
<author><name>Kill the Newsletter!</name></author>
$${entries.map(
(entry) => html`
<entry>
<id>urn:kill-the-newsletter:${entry.reference}</id>
<title>${entry.title}</title>
<author><name>${entry.author}</name></author>
<updated
>${new Date(entry.createdAt).toISOString()}</updated
>
<link
rel="alternate"
type="text/html"
href="${webApplication.get(
"url"
)}/alternates/${entry.reference}.html"
/>
<content type="html">${entry.content}</content>
</entry>
`
)}
</feed>
`.trim()
);
}
);
2021-03-11 11:29:05 +01:00
webApplication.get<{ entryReference: string }, HTML, {}, {}, {}>(
"/alternates/:entryReference.html",
(req, res, next) => {
const entry = database.get<{ content: string }>(
sql`SELECT "content" FROM "entries" WHERE "reference" = ${req.params.entryReference}`
);
if (entry === undefined) return next();
res.header("X-Robots-Tag", "noindex").send(entry.content);
}
);
// TODO: 404
2021-03-10 11:30:08 +01:00
const emailApplication = new SMTPServer();
2021-03-11 11:04:09 +01:00
function newReference(): string {
return cryptoRandomString({
length: 16,
characters: "abcdefghijklmnopqrstuvwxyz0123456789",
});
}
2021-03-10 11:30:08 +01:00
return { webApplication, emailApplication };
}
if (require.main === module) {
console.log(`Kill the Newsletter!/${VERSION}`);
const configurationFile = path.resolve(
process.argv[2] ?? path.join(process.cwd(), "configuration.js")
);
require(configurationFile)(require);
console.log(`Configuration file loaded from ‘${configurationFile}’.`);
}
/*
Change some dependencies
2020-05-05 08:12:57 +02:00
export const emailServer = new SMTPServer({
disabledCommands: ["AUTH", "STARTTLS"],
async onData(stream, session, callback) {
try {
const email = await mailparser.simpleParser(stream);
Add support for alternate URL
2020-07-14 19:42:41 +02:00
const content =
typeof email.html === "string" ? email.html : email.textAsHtml ?? "";
Close #31
2020-12-07 23:56:00 +01:00
for (const address of new Set(
session.envelope.rcptTo.map(({ address }) => address)
)) {
Change some dependencies
2020-05-05 08:12:57 +02:00
const match = address.match(
2021-03-10 11:30:08 +01:00
new RegExp(
`^(?<identifier>\\w+)@${escapeStringRegexp(EMAIL_DOMAIN)}$`
)
Change some dependencies
2020-05-05 08:12:57 +02:00
);
if (match?.groups === undefined) continue;
Fixes & refactorings
2020-07-23 17:11:41 +02:00
const identifier = match.groups.identifier.toLowerCase();
Don’t store alternates explicitly to save disk space Instead, fetch alternates from within the feed on the demand. This makes alternates marginally more expensive to retrieve, but saves on storage (which we were running out on the DigitalOcean deployment), and is a cleaner architecture overall: no need to keep the feeds and alternates in sync. Here’s a script to migrate existing feeds: // Call me with, for example: env "BASE_URL=https://kill-the-newsletter.com" npx ts-node migrate.ts // I’m idempotent and reentrant, you may call me multiple times if necessary (for example, if the migration fails in the middle for whatever reason) import { promises as fs } from "fs"; import path from "path"; import { JSDOM } from "jsdom"; const BASE_URL = process.env.BASE_URL ?? "http://localhost:8000"; const FEEDS_PATH = "static/feeds"; (async () => { await fs.rmdir("static/alternate", { recursive: true }); for (const feedPath of (await fs.readdir(FEEDS_PATH)).filter((feedPath) => feedPath.endsWith(".xml") )) { const text = await fs.readFile(path.join(FEEDS_PATH, feedPath), "utf-8"); const feed = new JSDOM(text, { contentType: "text/xml" }); const document = feed.window.document; const feedIdentifier = document .querySelector("id")! .textContent!.split(":")[2]; for (const entry of document.querySelectorAll("entry")) { const entryIdentifier = entry .querySelector("id")! .textContent!.split(":")[2]; entry .querySelector(`link[rel="alternate"]`) ?.setAttribute( "href", `${BASE_URL}/alternate/${feedIdentifier}/${entryIdentifier}.html` ); } await fs.writeFile( path.join(FEEDS_PATH, feedPath), `<?xml version="1.0" encoding="utf-8"?>${feed.serialize()}`.trim() ); console.log(feedIdentifier); } })();
2020-11-24 18:12:14 +01:00
const path = feedFilePath(identifier);
let text;
try {
text = await fs.readFile(path, "utf8");
} catch {
continue;
}
const feed = new JSDOM(text, { contentType: "text/xml" });
const document = feed.window.document;
const updated = document.querySelector("feed > updated");
if (updated === null) {
console.error(`Field ‘updated’ not found: ‘${path}’`);
continue;
}
updated.textContent = now();
const renderedEntry = entry(
Fixes & refactorings
2020-07-23 17:11:41 +02:00
identifier,
Don’t store alternates explicitly to save disk space Instead, fetch alternates from within the feed on the demand. This makes alternates marginally more expensive to retrieve, but saves on storage (which we were running out on the DigitalOcean deployment), and is a cleaner architecture overall: no need to keep the feeds and alternates in sync. Here’s a script to migrate existing feeds: // Call me with, for example: env "BASE_URL=https://kill-the-newsletter.com" npx ts-node migrate.ts // I’m idempotent and reentrant, you may call me multiple times if necessary (for example, if the migration fails in the middle for whatever reason) import { promises as fs } from "fs"; import path from "path"; import { JSDOM } from "jsdom"; const BASE_URL = process.env.BASE_URL ?? "http://localhost:8000"; const FEEDS_PATH = "static/feeds"; (async () => { await fs.rmdir("static/alternate", { recursive: true }); for (const feedPath of (await fs.readdir(FEEDS_PATH)).filter((feedPath) => feedPath.endsWith(".xml") )) { const text = await fs.readFile(path.join(FEEDS_PATH, feedPath), "utf-8"); const feed = new JSDOM(text, { contentType: "text/xml" }); const document = feed.window.document; const feedIdentifier = document .querySelector("id")! .textContent!.split(":")[2]; for (const entry of document.querySelectorAll("entry")) { const entryIdentifier = entry .querySelector("id")! .textContent!.split(":")[2]; entry .querySelector(`link[rel="alternate"]`) ?.setAttribute( "href", `${BASE_URL}/alternate/${feedIdentifier}/${entryIdentifier}.html` ); } await fs.writeFile( path.join(FEEDS_PATH, feedPath), `<?xml version="1.0" encoding="utf-8"?>${feed.serialize()}`.trim() ); console.log(feedIdentifier); } })();
2020-11-24 18:12:14 +01:00
createIdentifier(),
X(email.subject ?? ""),
X(email.from?.text ?? ""),
X(content)
);
const firstEntry = document.querySelector("feed > entry:first-of-type");
if (firstEntry === null)
document
.querySelector("feed")!
.insertAdjacentHTML("beforeend", renderedEntry);
else firstEntry.insertAdjacentHTML("beforebegin", renderedEntry);
while (feed.serialize().length > 500_000) {
const lastEntry = document.querySelector("feed > entry:last-of-type");
if (lastEntry === null) break;
lastEntry.remove();
}
await writeFileAtomic(
path,
html`<?xml version="1.0" encoding="utf-8"?>${feed.serialize()}`.trim()
);
Change some dependencies
2020-05-05 08:12:57 +02:00
}
callback();
} catch (error) {
console.error(
Fixes & refactorings
2020-07-23 17:11:41 +02:00
`Failed to receive message: ‘${JSON.stringify(session, null, 2)}’`
Change some dependencies
2020-05-05 08:12:57 +02:00
);
console.error(error);
stream.resume();
callback(new Error("Failed to receive message. Please try again."));
}
},
}).listen(EMAIL_PORT);
2021-03-10 11:30:08 +01:00
*/