#!/usr/bin/env node import assert from "node:assert/strict"; import fs from "node:fs/promises"; import path from "node:path"; import url from "node:url"; import timers from "node:timers/promises"; import os from "node:os"; import * as commander from "commander"; import express from "express"; import nodemailer from "nodemailer"; import lodash from "lodash"; import { execa, ExecaChildProcess } from "execa"; import caddyfile from "dedent"; import dedent from "dedent"; if (process.env.TEST === "kill-the-newsletter") { delete process.env.TEST; // import { jest, test, expect } from "@jest/globals"; // import os from "os"; // import path from "path"; // import fs from "fs"; // import * as got from "got"; // import nodemailer from "nodemailer"; // import html from "@leafac/html"; // import killTheNewsletter from "."; // jest.setTimeout(300_000); // test("Kill the Newsletter!", async () => { // // Start servers // const rootDirectory = fs.mkdtempSync( // path.join(os.tmpdir(), "kill-the-newsletter--test--") // ); // const { webApplication, emailApplication } = killTheNewsletter(rootDirectory); // const webServer = webApplication.listen( // new URL(webApplication.get("url")).port // ); // const emailServer = emailApplication.listen( // new URL(webApplication.get("email")).port // ); // const webClient = got.default.extend({ // prefixUrl: webApplication.get("url"), // }); // const emailClient = nodemailer.createTransport(webApplication.get("email")); // const emailHostname = new URL(webApplication.get("url")).hostname; // // Create feed // const create = (await webClient.post("", { form: { name: "A newsletter" } })) // .body; // expect(create).toMatch(`“A newsletter” inbox created`); // const feedReference = create.match(/\/feeds\/([a-z0-9]{16})\.xml/)![1]; // // Test feed properties // const feedOriginal = await webClient.get(`feeds/${feedReference}.xml`); // expect(feedOriginal.headers["content-type"]).toMatch("application/atom+xml"); // expect(feedOriginal.headers["x-robots-tag"]).toBe("noindex"); // expect(feedOriginal.body).toMatch(html`A newsletter`); // // Test alternate // const alternateReference = feedOriginal.body.match( // /\/alternates\/([a-z0-9]{16})\.html/ // )![1]; // const alternate = await webClient.get( // `alternates/${alternateReference}.html` // ); // expect(alternate.headers["content-type"]).toMatch("text/html"); // expect(alternate.headers["x-robots-tag"]).toBe("noindex"); // expect(alternate.body).toMatch(`Enjoy your readings!`); // // Test email with HTML // await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for a second to test that the ‘’ field will be updated // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email with HTML", // html: html`

Some HTML

`, // }); // const feedWithHTMLEntry = (await webClient.get(`feeds/${feedReference}.xml`)) // .body; // expect(feedWithHTMLEntry.match(/(.+?)<\/updated>/)![1]).not.toBe( // feedOriginal.body.match(/(.+?)<\/updated>/)![1] // ); // expect(feedWithHTMLEntry).toMatch( // html`publisher@example.com` // ); // expect(feedWithHTMLEntry).toMatch(html`Test email with HTML`); // expect(feedWithHTMLEntry).toMatch( // // prettier-ignore // html`${`

Some HTML

`}\n
` // ); // // Test email with text // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email with text", // text: "A link: https://kill-the-newsletter.com", // }); // expect((await webClient.get(`feeds/${feedReference}.xml`)).body).toMatch( // // prettier-ignore // html`${`

A link: https://kill-the-newsletter.com

`}
` // ); // // Test email missing fields // await emailClient.sendMail({ // to: `${feedReference}@${emailHostname}`, // }); // const feedMissingFields = (await webClient.get(`feeds/${feedReference}.xml`)) // .body; // expect(feedMissingFields).toMatch(html``); // expect(feedMissingFields).toMatch(html``); // expect(feedMissingFields).toMatch(html``); // // Test email to nonexistent ‘to’ (gets ignored) // await emailClient.sendMail({ // from: "publisher@example.com", // to: `nonexistent@${emailHostname}`, // subject: "Test email to nonexistent ‘to’ (gets ignored)", // text: "A link: https://kill-the-newsletter.com", // }); // expect((await webClient.get(`feeds/${feedReference}.xml`)).body).not.toMatch( // "Test email to nonexistent ‘to’ (gets ignored)" // ); // // Test truncation // for (let index = 1; index <= 5; index++) // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: `Test truncation: ${index}`, // text: `TRUNCATION ${index} `.repeat(10_000), // }); // const feedTruncated = (await webClient.get(`feeds/${feedReference}.xml`)) // .body; // expect(feedTruncated).toMatch("TRUNCATION 5"); // expect(feedTruncated).not.toMatch("TRUNCATION 1"); // // Test email that’s too long // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email that’s too long", // text: `TOO LONG `.repeat(100_000), // }); // const feedEvenMoreTruncated = ( // await webClient.get(`feeds/${feedReference}.xml`) // ).body; // expect(feedEvenMoreTruncated).not.toMatch("TOO LONG"); // expect(feedEvenMoreTruncated).not.toMatch("TRUNCATION 5"); // // Test email after truncation // await emailClient.sendMail({ // from: "publisher@example.com", // to: `${feedReference}@${emailHostname}`, // subject: "Test email after truncation", // text: "A link: https://kill-the-newsletter.com", // }); // expect((await webClient.get(`feeds/${feedReference}.xml`)).body).toMatch( // // prettier-ignore // html`Test email after truncation` // ); // // Stop servers // webServer.close(); // emailServer.close(); // }); assert.equal(1 + 1, 2); process.exit(0); } const version = JSON.parse( await fs.readFile(new URL("../../package.json", import.meta.url), "utf8") ).version; await commander.program .name("kill-the-newsletter") .description("Convert email newsletters into Atom feeds") .addOption( new commander.Option("--process-type ") .default("main") .hideHelp() ) .addOption( new commander.Option("--process-number ").hideHelp() ) .argument( "[configuration]", "Path to configuration file. If you don’t provide a configuration file, the application runs in demonstration mode.", url.fileURLToPath( new URL("../../configuration/default.mjs", import.meta.url) ) ) .version(version) .addHelpText( "after", "\n" + dedent` Configuration: See ‘https://github.com/courselore/courselore/blob/main/documentation/self-hosting.md’ for instructions, and ‘https://github.com/courselore/courselore/blob/main/configuration/example.mjs’ for an example. ` ) .allowExcessArguments(false) .showHelpAfterError() .action( async ( configuration: string, { processType, processNumber, }: { processType: "main" | "server" | "worker"; processNumber: string; } ) => { const stop = new Promise((resolve) => { const processKeepAlive = new AbortController(); timers .setInterval(1 << 30, undefined, { signal: processKeepAlive.signal, }) [Symbol.asyncIterator]() .next() .catch(() => {}); for (const event of [ "exit", "SIGHUP", "SIGINT", "SIGQUIT", "SIGTERM", "SIGUSR2", "SIGBREAK", ]) process.on(event, () => { processKeepAlive.abort(); resolve(); }); }); const application = { name: "courselore", version, process: { id: Math.random().toString(36).slice(2), type: processType, number: typeof processNumber === "string" ? Number(processNumber) : undefined, }, configuration: (await import(url.pathToFileURL(configuration).href)) .default, static: JSON.parse( await fs.readFile( new URL("../static/paths.json", import.meta.url), "utf8" ) ), ports: { server: lodash.times( os.cpus().length, (processNumber) => 6000 + processNumber ), serverEvents: lodash.times( os.cpus().length, (processNumber) => 7000 + processNumber ), workerEvents: lodash.times( os.cpus().length, (processNumber) => 8000 + processNumber ), }, addresses: { canonicalHostname: "courselore.org", metaCourseloreInvitation: "https://meta.courselore.org", tryHostname: "try.courselore.org", }, server: express() as any, serverEvents: express() as any, workerEvents: express() as any, } as Application; application.configuration.environment ??= "production"; application.configuration.demonstration ??= application.configuration.environment !== "production"; application.configuration.tunnel ??= false; application.configuration.alternativeHostnames ??= []; application.configuration.hstsPreload ??= false; application.configuration.caddy ??= caddyfile``; // #!/usr/bin/env node // import path from "path"; // import express from "express"; // import { SMTPServer } from "smtp-server"; // import mailparser from "mailparser"; // import fs from "fs-extra"; // import cryptoRandomString from "crypto-random-string"; // import { html, HTML } from "@leafac/html"; // import { css, process as processCSS } from "@leafac/css"; // import javascript from "tagged-template-noop"; // import { sql, Database } from "@leafac/sqlite"; // import databaseMigrate from "@leafac/sqlite-migration"; // const VERSION = require("../package.json").version; // export default function killTheNewsletter( // rootDirectory: string // ): { webApplication: express.Express; emailApplication: SMTPServer } { // const webApplication = express(); // webApplication.set("url", "http://localhost:4000"); // webApplication.set("email", "smtp://localhost:2525"); // webApplication.set("administrator", "mailto:kill-the-newsletter@leafac.com"); // fs.ensureDirSync(rootDirectory); // const database = new Database( // path.join(rootDirectory, "kill-the-newsletter.db") // ); // databaseMigrate(database, [ // sql` // CREATE TABLE "feeds" ( // "id" INTEGER PRIMARY KEY AUTOINCREMENT, // "createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, // "updatedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, // "reference" TEXT NOT NULL UNIQUE, // "title" TEXT NOT NULL // ); // CREATE TABLE "entries" ( // "id" INTEGER PRIMARY KEY AUTOINCREMENT, // "createdAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, // "reference" TEXT NOT NULL UNIQUE, // "feed" INTEGER NOT NULL REFERENCES "feeds", // "title" TEXT NOT NULL, // "author" TEXT NOT NULL, // "content" TEXT NOT NULL // ); // `, // sql` // CREATE INDEX "entriesFeed" ON "entries" ("feed"); // `, // ]); // webApplication.use(express.static(path.join(__dirname, "../public"))); // webApplication.use(express.urlencoded({ extended: true })); // const logo = fs.readFileSync(path.join(__dirname, "../public/logo.svg")); // function layout(body: HTML): HTML { // return processCSS(html` // // // // // // // // Kill the Newsletter! // // //
//

// Kill the Newsletter! //

//

Convert email newsletters into Atom feeds

//

// $${logo} //

//
//
$${body}
// // // // // `); // } // webApplication.get<{}, HTML, {}, {}, {}>("/", (req, res) => { // res.send( // layout(html` //
//

// //

//

// //

//
// `) // ); // }); // webApplication.post<{}, HTML, { name?: string }, {}, {}>("/", (req, res) => { // if ( // typeof req.body.name !== "string" || // req.body.name.trim() === "" || // req.body.name.length > 500 // ) // return res.status(422).send( // layout( // html` //

// Error: Missing newsletter name. // Try again. //

// ` // ) // ); // const feedReference = newReference(); // const welcomeTitle = `“${req.body.name}” inbox created`; // const welcomeContent = html` //

// Sign up for the newsletter with
// ${feedReference}@${new URL(webApplication.get("email")) // .hostname} //

//

// Subscribe to the Atom feed at
// ${webApplication.get("url")}/feeds/${feedReference}.xml //

//

// Don’t share these addresses.
// They contain an identifier that other people could use to send you spam // and to control your newsletter subscriptions. //

//

Enjoy your readings!

//

// Create another inbox //

// `; // database.executeTransaction(() => { // const feedId = database.run( // sql`INSERT INTO "feeds" ("reference", "title") VALUES (${feedReference}, ${req.body.name})` // ).lastInsertRowid; // database.run( // sql` // INSERT INTO "entries" ("reference", "feed", "title", "author", "content") // VALUES ( // ${newReference()}, // ${feedId}, // ${welcomeTitle}, // ${"Kill the Newsletter!"}, // ${welcomeContent} // ) // ` // ); // }); // res.send( // layout(html` //

${welcomeTitle}

// $${welcomeContent} // `) // ); // }); // function renderFeed(feedReference: string): HTML | undefined { // const feed = database.get<{ // id: number; // updatedAt: string; // title: string; // }>( // sql`SELECT "id", "updatedAt", "title" FROM "feeds" WHERE "reference" = ${feedReference}` // ); // if (feed === undefined) return; // const entries = database.all<{ // createdAt: string; // reference: string; // title: string; // author: string; // content: string; // }>( // sql` // SELECT "createdAt", "reference", "title", "author", "content" // FROM "entries" // WHERE "feed" = ${feed.id} // ORDER BY "id" DESC // ` // ); // return html` // // // // // urn:kill-the-newsletter:${feedReference} // ${feed.title} // Kill the Newsletter! Inbox: // ${feedReference}@${new URL(webApplication.get("email")).hostname} → // ${webApplication.get("url")}/feeds/${feedReference}.xml // ${new Date(feed.updatedAt).toISOString()} // Kill the Newsletter! // $${entries.map( // (entry) => html` // // urn:kill-the-newsletter:${entry.reference} // ${entry.title} // ${entry.author} // ${new Date(entry.createdAt).toISOString()} // // ${entry.content} // // ` // )} // // `.trim(); // } // webApplication.get<{ feedReference: string }, HTML, {}, {}, {}>( // "/feeds/:feedReference.xml", // (req, res, next) => { // const feed = renderFeed(req.params.feedReference); // if (feed === undefined) return next(); // res.type("atom").header("X-Robots-Tag", "noindex").send(feed); // } // ); // webApplication.get<{ entryReference: string }, HTML, {}, {}, {}>( // "/alternates/:entryReference.html", // (req, res, next) => { // const entry = database.get<{ content: string }>( // sql`SELECT "content" FROM "entries" WHERE "reference" = ${req.params.entryReference}` // ); // if (entry === undefined) return next(); // res.header("X-Robots-Tag", "noindex").send(entry.content); // } // ); // webApplication.use((req, res) => { // res.send( // layout(html` //

404 Not found

//

// Create a new inbox //

// `) // ); // }); // const emailApplication = new SMTPServer({ // disabledCommands: ["AUTH", "STARTTLS"], // async onData(stream, session, callback) { // try { // const email = await mailparser.simpleParser(stream); // const from = email.from?.text ?? ""; // const subject = email.subject ?? ""; // const body = // typeof email.html === "string" ? email.html : email.textAsHtml ?? ""; // database.executeTransaction(() => { // for (const address of new Set( // session.envelope.rcptTo.map( // (smtpServerAddress) => smtpServerAddress.address // ) // )) { // const addressParts = address.split("@"); // if (addressParts.length !== 2) continue; // const [feedReference, hostname] = addressParts; // if (hostname !== new URL(webApplication.get("email")).hostname) // continue; // const feed = database.get<{ id: number }>( // sql`SELECT "id" FROM "feeds" WHERE "reference" = ${feedReference}` // ); // if (feed === undefined) continue; // database.run( // sql` // INSERT INTO "entries" ("reference", "feed", "title", "author", "content") // VALUES ( // ${newReference()}, // ${feed.id}, // ${subject}, // ${from}, // ${body} // ) // ` // ); // database.run( // sql`UPDATE "feeds" SET "updatedAt" = CURRENT_TIMESTAMP WHERE "id" = ${feed.id}` // ); // while (renderFeed(feedReference)!.length > 500_000) // database.run( // sql`DELETE FROM "entries" WHERE "feed" = ${feed.id} ORDER BY "id" ASC LIMIT 1` // ); // } // }); // callback(); // } catch (error) { // console.error( // `Failed to receive message: ‘${JSON.stringify(session, null, 2)}’` // ); // console.error(error); // stream.resume(); // callback(new Error("Failed to receive message. Please try again.")); // } // }, // }); // function newReference(): string { // return cryptoRandomString({ // length: 16, // characters: "abcdefghijklmnopqrstuvwxyz0123456789", // }); // } // return { webApplication, emailApplication }; // } // if (require.main === module) { // console.log(`Kill the Newsletter!/${VERSION}`); // if (process.argv[2] === undefined) { // const { webApplication, emailApplication } = killTheNewsletter( // path.join(process.cwd(), "data") // ); // webApplication.listen(new URL(webApplication.get("url")).port, () => { // console.log(`Web server started at ${webApplication.get("url")}`); // }); // emailApplication.listen(new URL(webApplication.get("email")).port, () => { // console.log(`Email server started at ${webApplication.get("email")}`); // }); // } else { // const configurationFile = path.resolve(process.argv[2]); // require(configurationFile)(require); // console.log(`Configuration loaded from ‘${configurationFile}’.`); // } // } switch (application.process.type) { case "main": { const childProcesses = new Set(); let restartChildProcesses = true; for (const execaArguments of [ ...["server", "worker"].flatMap((processType) => lodash.times(os.cpus().length, (processNumber) => ({ file: process.argv[0], arguments: [ process.argv[1], "--process-type", processType, "--process-number", processNumber, configuration, ], options: { preferLocal: true, stdio: "inherit", ...(application.configuration.environment === "production" ? { env: { NODE_ENV: "production" } } : {}), }, })) ), { file: "caddy", arguments: ["run", "--config", "-", "--adapter", "caddyfile"], options: { preferLocal: true, stdout: "ignore", stderr: "ignore", input: caddyfile` { admin off ${ application.configuration.environment === "production" ? `email ${application.configuration.administratorEmail}` : `local_certs` } } (common) { header Cache-Control no-store header Content-Security-Policy "default-src https://${ application.configuration.hostname }/ 'unsafe-inline' 'unsafe-eval'; frame-ancestors 'none'; object-src 'none'" header Cross-Origin-Embedder-Policy require-corp header Cross-Origin-Opener-Policy same-origin header Cross-Origin-Resource-Policy same-origin header Referrer-Policy no-referrer header Strict-Transport-Security "max-age=31536000; includeSubDomains${ application.configuration.hstsPreload ? `; preload` : `` }" header X-Content-Type-Options nosniff header Origin-Agent-Cluster "?1" header X-DNS-Prefetch-Control off header X-Frame-Options DENY header X-Permitted-Cross-Domain-Policies none header -Server header -X-Powered-By header X-XSS-Protection 0 header Permissions-Policy "interest-cohort=()" encode zstd gzip } ${[ application.configuration.tunnel ? [] : [application.configuration.hostname], ...application.configuration.alternativeHostnames, ] .map((hostname) => `http://${hostname}`) .join(", ")} { import common redir https://{host}{uri} 308 handle_errors { import common } } ${ application.configuration.alternativeHostnames.length > 0 ? caddyfile` ${application.configuration.alternativeHostnames .map((hostname) => `https://${hostname}`) .join(", ")} { import common redir https://${ application.configuration.hostname }{uri} 307 handle_errors { import common } } ` : `` } http${application.configuration.tunnel ? `` : `s`}://${ application.configuration.hostname } { route { import common route { root * ${JSON.stringify( url.fileURLToPath( new URL("../static/", import.meta.url) ) )} @file_exists file route @file_exists { header Cache-Control "public, max-age=31536000, immutable" file_server } } route /files/* { root * ${JSON.stringify( path.resolve(application.configuration.dataDirectory) )} @file_exists file route @file_exists { header Cache-Control "private, max-age=31536000, immutable" @must_be_downloaded not path *.png *.jpg *.jpeg *.gif *.mp3 *.mp4 *.m4v *.ogg *.mov *.mpeg *.avi *.pdf *.txt header @must_be_downloaded Content-Disposition attachment @may_be_embedded_in_other_sites path *.png *.jpg *.jpeg *.gif *.mp3 *.mp4 *.m4v *.ogg *.mov *.mpeg *.avi *.pdf header @may_be_embedded_in_other_sites Cross-Origin-Resource-Policy cross-origin file_server } } reverse_proxy ${application.ports.server .map((port) => `127.0.0.1:${port}`) .join(" ")} } handle_errors { import common } } ${application.configuration.caddy} `, }, }, ]) (async () => { while (restartChildProcesses) { const childProcess = execa( execaArguments.file, execaArguments.arguments as any, { ...execaArguments.options, reject: false, cleanup: false, } as any ); childProcesses.add(childProcess); const childProcessResult = await childProcess; application.log( "CHILD PROCESS RESULT", JSON.stringify(childProcessResult, undefined, 2) ); childProcesses.delete(childProcess); } })(); await stop; restartChildProcesses = false; for (const childProcess of childProcesses) childProcess.cancel(); break; } case "server": { const serverApplication = application.server; const eventsApplication = application.serverEvents; serverApplication.emit("start"); eventsApplication.emit("start"); const server = serverApplication.listen( application.ports.server[application.process.number], "127.0.0.1" ); const events = eventsApplication.listen( application.ports.serverEvents[application.process.number], "127.0.0.1" ); await stop; server.close(); events.close(); serverApplication.emit("stop"); eventsApplication.emit("stop"); break; } case "worker": { const eventsApplication = application.workerEvents; eventsApplication.emit("start"); const events = eventsApplication.listen( application.ports.workerEvents[application.process.number], "127.0.0.1" ); await stop; events.close(); eventsApplication.emit("stop"); break; } } await timers.setTimeout(10 * 1000, undefined, { ref: false }); process.exit(1); } ) .parseAsync();