#!/usr/bin/env node import assert from "node:assert/strict"; import fs from "node:fs/promises"; import path from "node:path"; import url from "node:url"; import timers from "node:timers/promises"; import os from "node:os"; import * as commander from "commander"; import express from "express"; import nodemailer from "nodemailer"; import { Database, sql } from "@leafac/sqlite"; import lodash from "lodash"; import { execa, ExecaChildProcess } from "execa"; import caddyfile from "dedent"; import dedent from "dedent"; if (process.env.TEST === "kill-the-newsletter") { delete process.env.TEST; // import { jest, test, expect } from "@jest/globals"; // import os from "os"; // import path from "path"; // import fs from "fs"; // import * as got from "got"; // import nodemailer from "nodemailer"; // import html from "@leafac/html"; // import killTheNewsletter from "."; // jest.setTimeout(300_000); // test("Kill the Newsletter!", async () => { // // Start servers // const rootDirectory = fs.mkdtempSync( // path.join(os.tmpdir(), "kill-the-newsletter--test--") // ); // const { webApplication, emailApplication } = killTheNewsletter(rootDirectory); // const webServer = webApplication.listen( // new URL(webApplication.get("url")).port // ); // const emailServer = emailApplication.listen( // new URL(webApplication.get("email")).port // ); // const webClient = got.default.extend({ // prefixUrl: webApplication.get("url"), // }); // const emailClient = nodemailer.createTransport(webApplication.get("email")); // const emailHostname = new URL(webApplication.get("url")).hostname; // // Create feed // const create = (await webClient.post("", { form: { name: "A newsletter" } })) // .body; // expect(create).toMatch(`“A newsletter” inbox created`); // const feedReference = create.match(/\/feeds\/([a-z0-9]{16})\.xml/)![1]; // // Test feed properties // const feedOriginal = await webClient.get(`feeds/${feedReference}.xml`); // expect(feedOriginal.headers["content-type"]).toMatch("application/atom+xml"); // expect(feedOriginal.headers["x-robots-tag"]).toBe("noindex"); // expect(feedOriginal.body).toMatch(html`A newsletter`); // // Test alternate // const alternateReference = feedOriginal.body.match( // /\/alternates\/([a-z0-9]{16})\.html/ // )![1]; // const alternate = await webClient.get( // `alternates/${alternateReference}.html` // ); // expect(alternate.headers["content-type"]).toMatch("text/html"); // expect(alternate.headers["x-robots-tag"]).toBe("noindex"); // expect(alternate.body).toMatch(`Enjoy your readings!`); // // Test email with HTML // await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for a second to test that the ‘’ field will be updated // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email with HTML", // html: html`

Some HTML

`, // }); // const feedWithHTMLEntry = (await webClient.get(`feeds/${feedReference}.xml`)) // .body; // expect(feedWithHTMLEntry.match(/(.+?)<\/updated>/)![1]).not.toBe( // feedOriginal.body.match(/(.+?)<\/updated>/)![1] // ); // expect(feedWithHTMLEntry).toMatch( // html`publisher@example.com` // ); // expect(feedWithHTMLEntry).toMatch(html`Test email with HTML`); // expect(feedWithHTMLEntry).toMatch( // // prettier-ignore // html`${`

Some HTML

`}\n
` // ); // // Test email with text // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email with text", // text: "A link: https://kill-the-newsletter.com", // }); // expect((await webClient.get(`feeds/${feedReference}.xml`)).body).toMatch( // // prettier-ignore // html`${`

A link: https://kill-the-newsletter.com

`}
` // ); // // Test email missing fields // await emailClient.sendMail({ // to: `${feedReference}@${emailHostname}`, // }); // const feedMissingFields = (await webClient.get(`feeds/${feedReference}.xml`)) // .body; // expect(feedMissingFields).toMatch(html``); // expect(feedMissingFields).toMatch(html``); // expect(feedMissingFields).toMatch(html``); // // Test email to nonexistent ‘to’ (gets ignored) // await emailClient.sendMail({ // from: "publisher@example.com", // to: `nonexistent@${emailHostname}`, // subject: "Test email to nonexistent ‘to’ (gets ignored)", // text: "A link: https://kill-the-newsletter.com", // }); // expect((await webClient.get(`feeds/${feedReference}.xml`)).body).not.toMatch( // "Test email to nonexistent ‘to’ (gets ignored)" // ); // // Test truncation // for (let index = 1; index <= 5; index++) // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: `Test truncation: ${index}`, // text: `TRUNCATION ${index} `.repeat(10_000), // }); // const feedTruncated = (await webClient.get(`feeds/${feedReference}.xml`)) // .body; // expect(feedTruncated).toMatch("TRUNCATION 5"); // expect(feedTruncated).not.toMatch("TRUNCATION 1"); // // Test email that’s too long // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email that’s too long", // text: `TOO LONG `.repeat(100_000), // }); // const feedEvenMoreTruncated = ( // await webClient.get(`feeds/${feedReference}.xml`) // ).body; // expect(feedEvenMoreTruncated).not.toMatch("TOO LONG"); // expect(feedEvenMoreTruncated).not.toMatch("TRUNCATION 5"); // // Test email after truncation // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email after truncation", // text: "A link: https://kill-the-newsletter.com", // }); // expect((await webClient.get(`feeds/${feedReference}.xml`)).body).toMatch( // // prettier-ignore // html`Test email after truncation` // ); // // Stop servers // webServer.close(); // emailServer.close(); // }); assert.equal(1 + 1, 2); process.exit(0); } const version = JSON.parse( await fs.readFile(new URL("../../package.json", import.meta.url), "utf8") ).version; await commander.program .name("kill-the-newsletter") .description("Convert email newsletters into Atom feeds") .addOption( new commander.Option("--process-type ") .default("main") .hideHelp() ) .addOption( new commander.Option("--process-number ").hideHelp() ) .argument( "[configuration]", "Path to configuration file. If you don’t provide a configuration file, the application runs in demonstration mode.", url.fileURLToPath( new URL("../../configuration/default.mjs", import.meta.url) ) ) .version(version) .addHelpText( "after", "\n" + dedent` Configuration: See ‘https://github.com/courselore/courselore/blob/main/documentation/self-hosting.md’ for instructions, and ‘https://github.com/courselore/courselore/blob/main/configuration/example.mjs’ for an example. ` ) .allowExcessArguments(false) .showHelpAfterError() .action( async ( configuration: string, { processType, processNumber, }: { processType: "main" | "web" | "email"; processNumber: string; } ) => { const stop = new Promise((resolve) => { const processKeepAlive = new AbortController(); timers .setInterval(1 << 30, undefined, { signal: processKeepAlive.signal, }) [Symbol.asyncIterator]() .next() .catch(() => {}); for (const event of [ "exit", "SIGHUP", "SIGINT", "SIGQUIT", "SIGTERM", "SIGUSR2", "SIGBREAK", ]) process.on(event, () => { processKeepAlive.abort(); resolve(); }); }); const application: { name: string; version: string; process: { id: string; type: "main" | "web" | "email"; number: number; }; configuration: { hostname: string; dataDirectory: string; administratorEmail: string; environment: "production" | "development" | "other"; tunnel: boolean; alternativeHostnames: string[]; hstsPreload: boolean; caddy: string; }; static: { [path: string]: string; }; ports: { web: number[]; }; web: Omit & Function; email: "TODO"; log(...messageParts: string[]): void; database: Database; } = { name: "kill-the-newsletter", version, process: { id: Math.random().toString(36).slice(2), type: processType, number: (typeof processNumber === "string" ? Number(processNumber) : undefined) as number, }, configuration: (await import(url.pathToFileURL(configuration).href)) .default, static: JSON.parse( await fs.readFile( new URL("../static/paths.json", import.meta.url), "utf8" ) ), ports: { web: lodash.times( os.cpus().length, (processNumber) => 6000 + processNumber ), }, web: express(), email: "TODO", } as any; application.configuration.environment ??= "production"; application.configuration.tunnel ??= false; application.configuration.alternativeHostnames ??= []; application.configuration.hstsPreload ??= false; application.configuration.caddy ??= caddyfile``; application.log = (...messageParts) => { console.log( [ new Date().toISOString(), application.process.type, application.process.number, application.process.id, ...messageParts, ].join(" \t") ); }; application.log( "STARTED", ...(application.process.type === "main" ? [ application.name, application.version, `https://${application.configuration.hostname}`, ] : []) ); process.once("exit", () => { application.log("STOPPED"); }); type ResponseLocalsLogging = { log(...messageParts: string[]): void; }; application.web.enable("trust proxy"); application.web.use<{}, any, {}, {}, ResponseLocalsLogging>( (request, response, next) => { if (response.locals.log !== undefined) return next(); const id = Math.random().toString(36).slice(2); const time = process.hrtime.bigint(); response.locals.log = (...messageParts) => { application.log( id, `${(process.hrtime.bigint() - time) / 1_000_000n}ms`, request.ip, request.method, request.originalUrl, ...messageParts ); }; const log = response.locals.log; log("STARTING..."); response.once("close", () => { const contentLength = response.getHeader("Content-Length"); log( "FINISHED", String(response.statusCode), ...(typeof contentLength === "string" ? [`${Math.ceil(Number(contentLength) / 1000)}kB`] : []) ); }); next(); } ); await fs.mkdir(application.configuration.dataDirectory, { recursive: true, }); application.database = new Database( path.join( application.configuration.dataDirectory, `${application.name}.db` ) ); process.once("exit", () => { application.database.close(); }); if (application.process.type === "main") { application.log("DATABASE MIGRATION", "STARTING..."); application.database.pragma("journal_mode = WAL"); // TODO: STOP USING DEFAULT VALUES. // TODO: DOUBLE-CHECK THAT THE OLD MIGRATION SYSTEM IS COMPATIBLE WITH THIS, USING SQLITE’S ‘PRAGMA USER_DATA’ await application.database.migrate( sql` CREATE TABLE "feeds" ( "id" INTEGER PRIMARY KEY AUTOINCREMENT, "createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, "reference" TEXT NOT NULL UNIQUE, "title" TEXT NOT NULL ); CREATE TABLE "entries" ( "id" INTEGER PRIMARY KEY AUTOINCREMENT, "createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, "reference" TEXT NOT NULL UNIQUE, "feed" INTEGER NOT NULL REFERENCES "feeds", "title" TEXT NOT NULL, "author" TEXT NOT NULL, "content" TEXT NOT NULL ); `, sql` CREATE INDEX "entriesFeed" ON "entries" ("feed"); ` ); application.log("DATABASE MIGRATION", "FINISHED"); } application.web.get<{}, any, {}, {}, ResponseLocalsLogging>( "/", (request, response) => { response.send("TODO"); } ); // #!/usr/bin/env node // import path from "path"; // import express from "express"; // import { SMTPServer } from "smtp-server"; // import mailparser from "mailparser"; // import fs from "fs-extra"; // import cryptoRandomString from "crypto-random-string"; // import { html, HTML } from "@leafac/html"; // import { css, process as processCSS } from "@leafac/css"; // import javascript from "tagged-template-noop"; // import { sql, Database } from "@leafac/sqlite"; // import databaseMigrate from "@leafac/sqlite-migration"; // const VERSION = require("../package.json").version; // export default function killTheNewsletter( // rootDirectory: string // ): { webApplication: express.Express; emailApplication: SMTPServer } { // const webApplication = express(); // webApplication.set("url", "http://localhost:4000"); // webApplication.set("email", "smtp://localhost:2525"); // webApplication.set("administrator", "mailto:kill-the-newsletter@leafac.com"); // fs.ensureDirSync(rootDirectory); // const database = new Database( // path.join(rootDirectory, "kill-the-newsletter.db") // ); // databaseMigrate(database, [ // ]); // webApplication.use(express.static(path.join(__dirname, "../public"))); // webApplication.use(express.urlencoded({ extended: true })); // const logo = fs.readFileSync(path.join(__dirname, "../public/logo.svg")); // function layout(body: HTML): HTML { // return processCSS(html` // // // // // // // // Kill the Newsletter! // // //
//

// Kill the Newsletter! //

//

Convert email newsletters into Atom feeds

//

// $${logo} //

//
//
$${body}
// // // // // `); // } // webApplication.get<{}, HTML, {}, {}, {}>("/", (req, res) => { // res.send( // layout(html` //
//

// //

//

// //

//
// `) // ); // }); // webApplication.post<{}, HTML, { name?: string }, {}, {}>("/", (req, res) => { // if ( // typeof req.body.name !== "string" || // req.body.name.trim() === "" || // req.body.name.length > 500 // ) // return res.status(422).send( // layout( // html` //

// Error: Missing newsletter name. // Try again. //

// ` // ) // ); // const feedReference = newReference(); // const welcomeTitle = `“${req.body.name}” inbox created`; // const welcomeContent = html` //

// Sign up for the newsletter with
// ${feedReference}@${new URL(webApplication.get("email")) // .hostname} //

//

// Subscribe to the Atom feed at
// ${webApplication.get("url")}/feeds/${feedReference}.xml //

//

// Don’t share these addresses.
// They contain an identifier that other people could use to send you spam // and to control your newsletter subscriptions. //

//

Enjoy your readings!

//

// Create another inbox //

// `; // database.executeTransaction(() => { // const feedId = database.run( // sql`INSERT INTO "feeds" ("reference", "title") VALUES (${feedReference}, ${req.body.name})` // ).lastInsertRowid; // database.run( // sql` // INSERT INTO "entries" ("reference", "feed", "title", "author", "content") // VALUES ( // ${newReference()}, // ${feedId}, // ${welcomeTitle}, // ${"Kill the Newsletter!"}, // ${welcomeContent} // ) // ` // ); // }); // res.send( // layout(html` //

${welcomeTitle}

// $${welcomeContent} // `) // ); // }); // function renderFeed(feedReference: string): HTML | undefined { // const feed = database.get<{ // id: number; // updatedAt: string; // title: string; // }>( // sql`SELECT "id", "updatedAt", "title" FROM "feeds" WHERE "reference" = ${feedReference}` // ); // if (feed === undefined) return; // const entries = database.all<{ // createdAt: string; // reference: string; // title: string; // author: string; // content: string; // }>( // sql` // SELECT "createdAt", "reference", "title", "author", "content" // FROM "entries" // WHERE "feed" = ${feed.id} // ORDER BY "id" DESC // ` // ); // return html` // // // // // urn:kill-the-newsletter:${feedReference} // ${feed.title} // Kill the Newsletter! Inbox: // ${feedReference}@${new URL(webApplication.get("email")).hostname} → // ${webApplication.get("url")}/feeds/${feedReference}.xml // ${new Date(feed.updatedAt).toISOString()} // Kill the Newsletter! // $${entries.map( // (entry) => html` // // urn:kill-the-newsletter:${entry.reference} // ${entry.title} // ${entry.author} // ${new Date(entry.createdAt).toISOString()} // // ${entry.content} // // ` // )} // // `.trim(); // } // webApplication.get<{ feedReference: string }, HTML, {}, {}, {}>( // "/feeds/:feedReference.xml", // (req, res, next) => { // const feed = renderFeed(req.params.feedReference); // if (feed === undefined) return next(); // res.type("atom").header("X-Robots-Tag", "noindex").send(feed); // } // ); // webApplication.get<{ entryReference: string }, HTML, {}, {}, {}>( // "/alternates/:entryReference.html", // (req, res, next) => { // const entry = database.get<{ content: string }>( // sql`SELECT "content" FROM "entries" WHERE "reference" = ${req.params.entryReference}` // ); // if (entry === undefined) return next(); // res.header("X-Robots-Tag", "noindex").send(entry.content); // } // ); // webApplication.use((req, res) => { // res.send( // layout(html` //

404 Not found

//

// Create a new inbox //

// `) // ); // }); // const emailApplication = new SMTPServer({ // disabledCommands: ["AUTH", "STARTTLS"], // async onData(stream, session, callback) { // try { // const email = await mailparser.simpleParser(stream); // const from = email.from?.text ?? ""; // const subject = email.subject ?? ""; // const body = // typeof email.html === "string" ? email.html : email.textAsHtml ?? ""; // database.executeTransaction(() => { // for (const address of new Set( // session.envelope.rcptTo.map( // (smtpServerAddress) => smtpServerAddress.address // ) // )) { // const addressParts = address.split("@"); // if (addressParts.length !== 2) continue; // const [feedReference, hostname] = addressParts; // if (hostname !== new URL(webApplication.get("email")).hostname) // continue; // const feed = database.get<{ id: number }>( // sql`SELECT "id" FROM "feeds" WHERE "reference" = ${feedReference}` // ); // if (feed === undefined) continue; // database.run( // sql` // INSERT INTO "entries" ("reference", "feed", "title", "author", "content") // VALUES ( // ${newReference()}, // ${feed.id}, // ${subject}, // ${from}, // ${body} // ) // ` // ); // database.run( // sql`UPDATE "feeds" SET "updatedAt" = CURRENT_TIMESTAMP WHERE "id" = ${feed.id}` // ); // while (renderFeed(feedReference)!.length > 500_000) // database.run( // sql`DELETE FROM "entries" WHERE "feed" = ${feed.id} ORDER BY "id" ASC LIMIT 1` // ); // } // }); // callback(); // } catch (error) { // console.error( // `Failed to receive message: ‘${JSON.stringify(session, null, 2)}’` // ); // console.error(error); // stream.resume(); // callback(new Error("Failed to receive message. Please try again.")); // } // }, // }); // function newReference(): string { // return cryptoRandomString({ // length: 16, // characters: "abcdefghijklmnopqrstuvwxyz0123456789", // }); // } // return { webApplication, emailApplication }; // } // if (require.main === module) { // console.log(`Kill the Newsletter!/${VERSION}`); // if (process.argv[2] === undefined) { // const { webApplication, emailApplication } = killTheNewsletter( // path.join(process.cwd(), "data") // ); // webApplication.listen(new URL(webApplication.get("url")).port, () => { // console.log(`Web server started at ${webApplication.get("url")}`); // }); // emailApplication.listen(new URL(webApplication.get("email")).port, () => { // console.log(`Email server started at ${webApplication.get("email")}`); // }); // } else { // const configurationFile = path.resolve(process.argv[2]); // require(configurationFile)(require); // console.log(`Configuration loaded from ‘${configurationFile}’.`); // } // } switch (application.process.type) { case "main": { const childProcesses = new Set(); let restartChildProcesses = true; for (const execaArguments of [ ...Object.entries({ web: os.cpus().length, email: 1 }).flatMap( ([processType, processCount]) => lodash.times(processCount, (processNumber) => ({ file: process.argv[0], arguments: [ process.argv[1], "--process-type", processType, "--process-number", processNumber, configuration, ], options: { preferLocal: true, stdio: "inherit", ...(application.configuration.environment === "production" ? { env: { NODE_ENV: "production" } } : {}), }, })) ), { file: "caddy", arguments: ["run", "--config", "-", "--adapter", "caddyfile"], options: { preferLocal: true, stdout: "ignore", stderr: "ignore", input: caddyfile` { admin off ${ application.configuration.environment === "production" ? `email ${application.configuration.administratorEmail}` : `local_certs` } } (common) { header Cache-Control no-store header Content-Security-Policy "default-src https://${ application.configuration.hostname }/ 'unsafe-inline' 'unsafe-eval'; frame-ancestors 'none'; object-src 'none'" header Cross-Origin-Embedder-Policy require-corp header Cross-Origin-Opener-Policy same-origin header Cross-Origin-Resource-Policy same-origin header Referrer-Policy no-referrer header Strict-Transport-Security "max-age=31536000; includeSubDomains${ application.configuration.hstsPreload ? `; preload` : `` }" header X-Content-Type-Options nosniff header Origin-Agent-Cluster "?1" header X-DNS-Prefetch-Control off header X-Frame-Options DENY header X-Permitted-Cross-Domain-Policies none header -Server header -X-Powered-By header X-XSS-Protection 0 header Permissions-Policy "interest-cohort=()" encode zstd gzip } ${[ ...(application.configuration.tunnel ? [] : [application.configuration.hostname]), ...application.configuration.alternativeHostnames, ] .map((hostname) => `http://${hostname}`) .join(", ")} { import common redir https://{host}{uri} 308 handle_errors { import common } } ${ application.configuration.alternativeHostnames.length > 0 ? caddyfile` ${application.configuration.alternativeHostnames .map((hostname) => `https://${hostname}`) .join(", ")} { import common redir https://${ application.configuration.hostname }{uri} 307 handle_errors { import common } } ` : `` } http${application.configuration.tunnel ? `` : `s`}://${ application.configuration.hostname } { route { import common route { root * ${JSON.stringify( url.fileURLToPath( new URL("../static/", import.meta.url) ) )} @file_exists file route @file_exists { header Cache-Control "public, max-age=31536000, immutable" file_server } } reverse_proxy ${application.ports.web .map((port) => `127.0.0.1:${port}`) .join(" ")} { lb_retries 1 } } handle_errors { import common } } ${application.configuration.caddy} `, }, }, ]) (async () => { while (restartChildProcesses) { const childProcess = execa( execaArguments.file, execaArguments.arguments as any, { ...execaArguments.options, reject: false, cleanup: false, } as any ); childProcesses.add(childProcess); const childProcessResult = await childProcess; application.log( "CHILD PROCESS RESULT", JSON.stringify(childProcessResult, undefined, 2) ); childProcesses.delete(childProcess); } })(); await stop; restartChildProcesses = false; for (const childProcess of childProcesses) childProcess.cancel(); break; } case "web": { const webApplication = application.web; webApplication.emit("start"); const server = webApplication.listen( application.ports.web[application.process.number], "127.0.0.1" ); await stop; server.close(); webApplication.emit("stop"); break; } case "email": { // TODO await stop; break; } } await timers.setTimeout(10 * 1000, undefined, { ref: false }); process.exit(1); } ) .parseAsync();