#!/usr/bin/env node
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import path from "node:path";
import url from "node:url";
import timers from "node:timers/promises";
import os from "node:os";
import * as commander from "commander";
import express from "express";
import nodemailer from "nodemailer";
import { Database, sql } from "@leafac/sqlite";
import lodash from "lodash";
import { execa, ExecaChildProcess } from "execa";
import caddyfile from "dedent";
import dedent from "dedent";
if (process.env.TEST === "kill-the-newsletter") {
delete process.env.TEST;
// import { jest, test, expect } from "@jest/globals";
// import os from "os";
// import path from "path";
// import fs from "fs";
// import * as got from "got";
// import nodemailer from "nodemailer";
// import html from "@leafac/html";
// import killTheNewsletter from ".";
// jest.setTimeout(300_000);
// test("Kill the Newsletter!", async () => {
// // Start servers
// const rootDirectory = fs.mkdtempSync(
// path.join(os.tmpdir(), "kill-the-newsletter--test--")
// );
// const { webApplication, emailApplication } = killTheNewsletter(rootDirectory);
// const webServer = webApplication.listen(
// new URL(webApplication.get("url")).port
// );
// const emailServer = emailApplication.listen(
// new URL(webApplication.get("email")).port
// );
// const webClient = got.default.extend({
// prefixUrl: webApplication.get("url"),
// });
// const emailClient = nodemailer.createTransport(webApplication.get("email"));
// const emailHostname = new URL(webApplication.get("url")).hostname;
// // Create feed
// const create = (await webClient.post("", { form: { name: "A newsletter" } }))
// .body;
// expect(create).toMatch(`“A newsletter” inbox created`);
// const feedReference = create.match(/\/feeds\/([a-z0-9]{16})\.xml/)![1];
// // Test feed properties
// const feedOriginal = await webClient.get(`feeds/${feedReference}.xml`);
// expect(feedOriginal.headers["content-type"]).toMatch("application/atom+xml");
// expect(feedOriginal.headers["x-robots-tag"]).toBe("noindex");
// expect(feedOriginal.body).toMatch(html`
A newsletter `);
// // Test alternate
// const alternateReference = feedOriginal.body.match(
// /\/alternates\/([a-z0-9]{16})\.html/
// )![1];
// const alternate = await webClient.get(
// `alternates/${alternateReference}.html`
// );
// expect(alternate.headers["content-type"]).toMatch("text/html");
// expect(alternate.headers["x-robots-tag"]).toBe("noindex");
// expect(alternate.body).toMatch(`Enjoy your readings!`);
// // Test email with HTML
// await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for a second to test that the ‘’ field will be updated
// await emailClient.sendMail({
// from: "publisher@example.com",
// to: `${feedReference}@${emailHostname}`,
// subject: "Test email with HTML",
// html: html`Some HTML
`,
// });
// const feedWithHTMLEntry = (await webClient.get(`feeds/${feedReference}.xml`))
// .body;
// expect(feedWithHTMLEntry.match(/(.+?)<\/updated>/)![1]).not.toBe(
// feedOriginal.body.match(/(.+?)<\/updated>/)![1]
// );
// expect(feedWithHTMLEntry).toMatch(
// html`publisher@example.com `
// );
// expect(feedWithHTMLEntry).toMatch(html`Test email with HTML `);
// expect(feedWithHTMLEntry).toMatch(
// // prettier-ignore
// html`${`Some HTML
`}\n `
// );
// // Test email with text
// await emailClient.sendMail({
// from: "publisher@example.com",
// to: `${feedReference}@${emailHostname}`,
// subject: "Test email with text",
// text: "A link: https://kill-the-newsletter.com",
// });
// expect((await webClient.get(`feeds/${feedReference}.xml`)).body).toMatch(
// // prettier-ignore
// html`${`A link: https://kill-the-newsletter.com
`} `
// );
// // Test email missing fields
// await emailClient.sendMail({
// to: `${feedReference}@${emailHostname}`,
// });
// const feedMissingFields = (await webClient.get(`feeds/${feedReference}.xml`))
// .body;
// expect(feedMissingFields).toMatch(html` `);
// expect(feedMissingFields).toMatch(html` `);
// expect(feedMissingFields).toMatch(html` `);
// // Test email to nonexistent ‘to’ (gets ignored)
// await emailClient.sendMail({
// from: "publisher@example.com",
// to: `nonexistent@${emailHostname}`,
// subject: "Test email to nonexistent ‘to’ (gets ignored)",
// text: "A link: https://kill-the-newsletter.com",
// });
// expect((await webClient.get(`feeds/${feedReference}.xml`)).body).not.toMatch(
// "Test email to nonexistent ‘to’ (gets ignored)"
// );
// // Test truncation
// for (let index = 1; index <= 5; index++)
// await emailClient.sendMail({
// from: "publisher@example.com",
// to: `${feedReference}@${emailHostname}`,
// subject: `Test truncation: ${index}`,
// text: `TRUNCATION ${index} `.repeat(10_000),
// });
// const feedTruncated = (await webClient.get(`feeds/${feedReference}.xml`))
// .body;
// expect(feedTruncated).toMatch("TRUNCATION 5");
// expect(feedTruncated).not.toMatch("TRUNCATION 1");
// // Test email that’s too long
// await emailClient.sendMail({
// from: "publisher@example.com",
// to: `${feedReference}@${emailHostname}`,
// subject: "Test email that’s too long",
// text: `TOO LONG `.repeat(100_000),
// });
// const feedEvenMoreTruncated = (
// await webClient.get(`feeds/${feedReference}.xml`)
// ).body;
// expect(feedEvenMoreTruncated).not.toMatch("TOO LONG");
// expect(feedEvenMoreTruncated).not.toMatch("TRUNCATION 5");
// // Test email after truncation
// await emailClient.sendMail({
// from: "publisher@example.com",
// to: `${feedReference}@${emailHostname}`,
// subject: "Test email after truncation",
// text: "A link: https://kill-the-newsletter.com",
// });
// expect((await webClient.get(`feeds/${feedReference}.xml`)).body).toMatch(
// // prettier-ignore
// html`Test email after truncation `
// );
// // Stop servers
// webServer.close();
// emailServer.close();
// });
assert.equal(1 + 1, 2);
process.exit(0);
}
const version = JSON.parse(
await fs.readFile(new URL("../../package.json", import.meta.url), "utf8")
).version;
await commander.program
.name("kill-the-newsletter")
.description("Convert email newsletters into Atom feeds")
.addOption(
new commander.Option("--process-type ")
.default("main")
.hideHelp()
)
.addOption(
new commander.Option("--process-number ").hideHelp()
)
.argument(
"[configuration]",
"Path to configuration file. If you don’t provide a configuration file, the application runs in demonstration mode.",
url.fileURLToPath(
new URL("../../configuration/default.mjs", import.meta.url)
)
)
.version(version)
.addHelpText(
"after",
"\n" +
dedent`
Configuration:
See ‘https://github.com/courselore/courselore/blob/main/documentation/self-hosting.md’ for instructions, and ‘https://github.com/courselore/courselore/blob/main/configuration/example.mjs’ for an example.
`
)
.allowExcessArguments(false)
.showHelpAfterError()
.action(
async (
configuration: string,
{
processType,
processNumber,
}: {
processType: "main" | "web" | "email";
processNumber: string;
}
) => {
const stop = new Promise((resolve) => {
const processKeepAlive = new AbortController();
timers
.setInterval(1 << 30, undefined, {
signal: processKeepAlive.signal,
})
[Symbol.asyncIterator]()
.next()
.catch(() => {});
for (const event of [
"exit",
"SIGHUP",
"SIGINT",
"SIGQUIT",
"SIGTERM",
"SIGUSR2",
"SIGBREAK",
])
process.on(event, () => {
processKeepAlive.abort();
resolve();
});
});
const application: {
name: string;
version: string;
process: {
id: string;
type: "main" | "web" | "email";
number: number;
};
configuration: {
hostname: string;
dataDirectory: string;
administratorEmail: string;
environment: "production" | "development" | "other";
tunnel: boolean;
alternativeHostnames: string[];
hstsPreload: boolean;
caddy: string;
};
static: {
[path: string]: string;
};
ports: {
web: number[];
};
web: Omit & Function;
email: "TODO";
log(...messageParts: string[]): void;
database: Database;
} = {
name: "kill-the-newsletter",
version,
process: {
id: Math.random().toString(36).slice(2),
type: processType,
number: (typeof processNumber === "string"
? Number(processNumber)
: undefined) as number,
},
configuration: (await import(url.pathToFileURL(configuration).href))
.default,
static: JSON.parse(
await fs.readFile(
new URL("../static/paths.json", import.meta.url),
"utf8"
)
),
ports: {
web: lodash.times(
os.cpus().length,
(processNumber) => 6000 + processNumber
),
},
web: express(),
email: "TODO",
} as any;
application.configuration.environment ??= "production";
application.configuration.tunnel ??= false;
application.configuration.alternativeHostnames ??= [];
application.configuration.hstsPreload ??= false;
application.configuration.caddy ??= caddyfile``;
application.log = (...messageParts) => {
console.log(
[
new Date().toISOString(),
application.process.type,
application.process.number,
application.process.id,
...messageParts,
].join(" \t")
);
};
application.log(
"STARTED",
...(application.process.type === "main"
? [
application.name,
application.version,
`https://${application.configuration.hostname}`,
]
: [])
);
process.once("exit", () => {
application.log("STOPPED");
});
type ResponseLocalsLogging = {
log(...messageParts: string[]): void;
};
application.web.enable("trust proxy");
application.web.use<{}, any, {}, {}, ResponseLocalsLogging>(
(request, response, next) => {
if (response.locals.log !== undefined) return next();
const id = Math.random().toString(36).slice(2);
const time = process.hrtime.bigint();
response.locals.log = (...messageParts) => {
application.log(
id,
`${(process.hrtime.bigint() - time) / 1_000_000n}ms`,
request.ip,
request.method,
request.originalUrl,
...messageParts
);
};
const log = response.locals.log;
log("STARTING...");
response.once("close", () => {
const contentLength = response.getHeader("Content-Length");
log(
"FINISHED",
String(response.statusCode),
...(typeof contentLength === "string"
? [`${Math.ceil(Number(contentLength) / 1000)}kB`]
: [])
);
});
next();
}
);
await fs.mkdir(application.configuration.dataDirectory, {
recursive: true,
});
application.database = new Database(
path.join(
application.configuration.dataDirectory,
`${application.name}.db`
)
);
process.once("exit", () => {
application.database.close();
});
if (application.process.type === "main") {
application.log("DATABASE MIGRATION", "STARTING...");
application.database.pragma("journal_mode = WAL");
// TODO: STOP USING DEFAULT VALUES.
// TODO: DOUBLE-CHECK THAT THE OLD MIGRATION SYSTEM IS COMPATIBLE WITH THIS, USING SQLITE’S ‘PRAGMA USER_DATA’
await application.database.migrate(
sql`
CREATE TABLE "feeds" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"reference" TEXT NOT NULL UNIQUE,
"title" TEXT NOT NULL
);
CREATE TABLE "entries" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
"reference" TEXT NOT NULL UNIQUE,
"feed" INTEGER NOT NULL REFERENCES "feeds",
"title" TEXT NOT NULL,
"author" TEXT NOT NULL,
"content" TEXT NOT NULL
);
`,
sql`
CREATE INDEX "entriesFeed" ON "entries" ("feed");
`
);
application.log("DATABASE MIGRATION", "FINISHED");
}
application.web.get<{}, any, {}, {}, ResponseLocalsLogging>(
"/",
(request, response) => {
response.send("TODO");
}
);
// #!/usr/bin/env node
// import path from "path";
// import express from "express";
// import { SMTPServer } from "smtp-server";
// import mailparser from "mailparser";
// import fs from "fs-extra";
// import cryptoRandomString from "crypto-random-string";
// import { html, HTML } from "@leafac/html";
// import { css, process as processCSS } from "@leafac/css";
// import javascript from "tagged-template-noop";
// import { sql, Database } from "@leafac/sqlite";
// import databaseMigrate from "@leafac/sqlite-migration";
// const VERSION = require("../package.json").version;
// export default function killTheNewsletter(
// rootDirectory: string
// ): { webApplication: express.Express; emailApplication: SMTPServer } {
// const webApplication = express();
// webApplication.set("url", "http://localhost:4000");
// webApplication.set("email", "smtp://localhost:2525");
// webApplication.set("administrator", "mailto:kill-the-newsletter@leafac.com");
// fs.ensureDirSync(rootDirectory);
// const database = new Database(
// path.join(rootDirectory, "kill-the-newsletter.db")
// );
// databaseMigrate(database, [
// ]);
// webApplication.use(express.static(path.join(__dirname, "../public")));
// webApplication.use(express.urlencoded({ extended: true }));
// const logo = fs.readFileSync(path.join(__dirname, "../public/logo.svg"));
// function layout(body: HTML): HTML {
// return processCSS(html`
//
//
//
//
//
//
//
// Kill the Newsletter!
//
//
//
// $${body}
//
//
//
//
// `);
// }
// webApplication.get<{}, HTML, {}, {}, {}>("/", (req, res) => {
// res.send(
// layout(html`
//
// `)
// );
// });
// webApplication.post<{}, HTML, { name?: string }, {}, {}>("/", (req, res) => {
// if (
// typeof req.body.name !== "string" ||
// req.body.name.trim() === "" ||
// req.body.name.length > 500
// )
// return res.status(422).send(
// layout(
// html`
//
// Error: Missing newsletter name.
// Try again .
//
// `
// )
// );
// const feedReference = newReference();
// const welcomeTitle = `“${req.body.name}” inbox created`;
// const welcomeContent = html`
//
// Sign up for the newsletter with
// ${feedReference}@${new URL(webApplication.get("email"))
// .hostname}
//
//
// Subscribe to the Atom feed at
// ${webApplication.get("url")}/feeds/${feedReference}.xml
//
//
// Don’t share these addresses.
// They contain an identifier that other people could use to send you spam
// and to control your newsletter subscriptions.
//
// Enjoy your readings!
//
// Create another inbox
//
// `;
// database.executeTransaction(() => {
// const feedId = database.run(
// sql`INSERT INTO "feeds" ("reference", "title") VALUES (${feedReference}, ${req.body.name})`
// ).lastInsertRowid;
// database.run(
// sql`
// INSERT INTO "entries" ("reference", "feed", "title", "author", "content")
// VALUES (
// ${newReference()},
// ${feedId},
// ${welcomeTitle},
// ${"Kill the Newsletter!"},
// ${welcomeContent}
// )
// `
// );
// });
// res.send(
// layout(html`
// ${welcomeTitle}
// $${welcomeContent}
// `)
// );
// });
// function renderFeed(feedReference: string): HTML | undefined {
// const feed = database.get<{
// id: number;
// updatedAt: string;
// title: string;
// }>(
// sql`SELECT "id", "updatedAt", "title" FROM "feeds" WHERE "reference" = ${feedReference}`
// );
// if (feed === undefined) return;
// const entries = database.all<{
// createdAt: string;
// reference: string;
// title: string;
// author: string;
// content: string;
// }>(
// sql`
// SELECT "createdAt", "reference", "title", "author", "content"
// FROM "entries"
// WHERE "feed" = ${feed.id}
// ORDER BY "id" DESC
// `
// );
// return html`
//
//
//
//
// urn:kill-the-newsletter:${feedReference}
// ${feed.title}
// Kill the Newsletter! Inbox:
// ${feedReference}@${new URL(webApplication.get("email")).hostname} →
// ${webApplication.get("url")}/feeds/${feedReference}.xml
// ${new Date(feed.updatedAt).toISOString()}
// Kill the Newsletter!
// $${entries.map(
// (entry) => html`
//
// urn:kill-the-newsletter:${entry.reference}
// ${entry.title}
// ${entry.author}
// ${new Date(entry.createdAt).toISOString()}
//
// ${entry.content}
//
// `
// )}
//
// `.trim();
// }
// webApplication.get<{ feedReference: string }, HTML, {}, {}, {}>(
// "/feeds/:feedReference.xml",
// (req, res, next) => {
// const feed = renderFeed(req.params.feedReference);
// if (feed === undefined) return next();
// res.type("atom").header("X-Robots-Tag", "noindex").send(feed);
// }
// );
// webApplication.get<{ entryReference: string }, HTML, {}, {}, {}>(
// "/alternates/:entryReference.html",
// (req, res, next) => {
// const entry = database.get<{ content: string }>(
// sql`SELECT "content" FROM "entries" WHERE "reference" = ${req.params.entryReference}`
// );
// if (entry === undefined) return next();
// res.header("X-Robots-Tag", "noindex").send(entry.content);
// }
// );
// webApplication.use((req, res) => {
// res.send(
// layout(html`
// 404 Not found
//
// Create a new inbox
//
// `)
// );
// });
// const emailApplication = new SMTPServer({
// disabledCommands: ["AUTH", "STARTTLS"],
// async onData(stream, session, callback) {
// try {
// const email = await mailparser.simpleParser(stream);
// const from = email.from?.text ?? "";
// const subject = email.subject ?? "";
// const body =
// typeof email.html === "string" ? email.html : email.textAsHtml ?? "";
// database.executeTransaction(() => {
// for (const address of new Set(
// session.envelope.rcptTo.map(
// (smtpServerAddress) => smtpServerAddress.address
// )
// )) {
// const addressParts = address.split("@");
// if (addressParts.length !== 2) continue;
// const [feedReference, hostname] = addressParts;
// if (hostname !== new URL(webApplication.get("email")).hostname)
// continue;
// const feed = database.get<{ id: number }>(
// sql`SELECT "id" FROM "feeds" WHERE "reference" = ${feedReference}`
// );
// if (feed === undefined) continue;
// database.run(
// sql`
// INSERT INTO "entries" ("reference", "feed", "title", "author", "content")
// VALUES (
// ${newReference()},
// ${feed.id},
// ${subject},
// ${from},
// ${body}
// )
// `
// );
// database.run(
// sql`UPDATE "feeds" SET "updatedAt" = CURRENT_TIMESTAMP WHERE "id" = ${feed.id}`
// );
// while (renderFeed(feedReference)!.length > 500_000)
// database.run(
// sql`DELETE FROM "entries" WHERE "feed" = ${feed.id} ORDER BY "id" ASC LIMIT 1`
// );
// }
// });
// callback();
// } catch (error) {
// console.error(
// `Failed to receive message: ‘${JSON.stringify(session, null, 2)}’`
// );
// console.error(error);
// stream.resume();
// callback(new Error("Failed to receive message. Please try again."));
// }
// },
// });
// function newReference(): string {
// return cryptoRandomString({
// length: 16,
// characters: "abcdefghijklmnopqrstuvwxyz0123456789",
// });
// }
// return { webApplication, emailApplication };
// }
// if (require.main === module) {
// console.log(`Kill the Newsletter!/${VERSION}`);
// if (process.argv[2] === undefined) {
// const { webApplication, emailApplication } = killTheNewsletter(
// path.join(process.cwd(), "data")
// );
// webApplication.listen(new URL(webApplication.get("url")).port, () => {
// console.log(`Web server started at ${webApplication.get("url")}`);
// });
// emailApplication.listen(new URL(webApplication.get("email")).port, () => {
// console.log(`Email server started at ${webApplication.get("email")}`);
// });
// } else {
// const configurationFile = path.resolve(process.argv[2]);
// require(configurationFile)(require);
// console.log(`Configuration loaded from ‘${configurationFile}’.`);
// }
// }
switch (application.process.type) {
case "main": {
const childProcesses = new Set();
let restartChildProcesses = true;
for (const execaArguments of [
...Object.entries({ web: os.cpus().length, email: 1 }).flatMap(
([processType, processCount]) =>
lodash.times(processCount, (processNumber) => ({
file: process.argv[0],
arguments: [
process.argv[1],
"--process-type",
processType,
"--process-number",
processNumber,
configuration,
],
options: {
preferLocal: true,
stdio: "inherit",
...(application.configuration.environment === "production"
? { env: { NODE_ENV: "production" } }
: {}),
},
}))
),
{
file: "caddy",
arguments: ["run", "--config", "-", "--adapter", "caddyfile"],
options: {
preferLocal: true,
stdout: "ignore",
stderr: "ignore",
input: caddyfile`
{
admin off
${
application.configuration.environment === "production"
? `email ${application.configuration.administratorEmail}`
: `local_certs`
}
}
(common) {
header Cache-Control no-store
header Content-Security-Policy "default-src https://${
application.configuration.hostname
}/ 'unsafe-inline' 'unsafe-eval'; frame-ancestors 'none'; object-src 'none'"
header Cross-Origin-Embedder-Policy require-corp
header Cross-Origin-Opener-Policy same-origin
header Cross-Origin-Resource-Policy same-origin
header Referrer-Policy no-referrer
header Strict-Transport-Security "max-age=31536000; includeSubDomains${
application.configuration.hstsPreload ? `; preload` : ``
}"
header X-Content-Type-Options nosniff
header Origin-Agent-Cluster "?1"
header X-DNS-Prefetch-Control off
header X-Frame-Options DENY
header X-Permitted-Cross-Domain-Policies none
header -Server
header -X-Powered-By
header X-XSS-Protection 0
header Permissions-Policy "interest-cohort=()"
encode zstd gzip
}
${[
...(application.configuration.tunnel
? []
: [application.configuration.hostname]),
...application.configuration.alternativeHostnames,
]
.map((hostname) => `http://${hostname}`)
.join(", ")} {
import common
redir https://{host}{uri} 308
handle_errors {
import common
}
}
${
application.configuration.alternativeHostnames.length > 0
? caddyfile`
${application.configuration.alternativeHostnames
.map((hostname) => `https://${hostname}`)
.join(", ")} {
import common
redir https://${
application.configuration.hostname
}{uri} 307
handle_errors {
import common
}
}
`
: ``
}
http${application.configuration.tunnel ? `` : `s`}://${
application.configuration.hostname
} {
route {
import common
route {
root * ${JSON.stringify(
url.fileURLToPath(
new URL("../static/", import.meta.url)
)
)}
@file_exists file
route @file_exists {
header Cache-Control "public, max-age=31536000, immutable"
file_server
}
}
reverse_proxy ${application.ports.web
.map((port) => `127.0.0.1:${port}`)
.join(" ")} {
lb_retries 1
}
}
handle_errors {
import common
}
}
${application.configuration.caddy}
`,
},
},
])
(async () => {
while (restartChildProcesses) {
const childProcess = execa(
execaArguments.file,
execaArguments.arguments as any,
{
...execaArguments.options,
reject: false,
cleanup: false,
} as any
);
childProcesses.add(childProcess);
const childProcessResult = await childProcess;
application.log(
"CHILD PROCESS RESULT",
JSON.stringify(childProcessResult, undefined, 2)
);
childProcesses.delete(childProcess);
}
})();
await stop;
restartChildProcesses = false;
for (const childProcess of childProcesses) childProcess.cancel();
break;
}
case "web": {
const webApplication = application.web;
webApplication.emit("start");
const server = webApplication.listen(
application.ports.web[application.process.number],
"127.0.0.1"
);
await stop;
server.close();
webApplication.emit("stop");
break;
}
case "email": {
// TODO
await stop;
break;
}
}
await timers.setTimeout(10 * 1000, undefined, { ref: false });
process.exit(1);
}
)
.parseAsync();