Move from xml2js to xmlbuilder2

This commit is contained in:
Leandro Facchinetti 2020-03-31 14:14:44 -04:00
parent e8a99001ed
commit e212793337
5 changed files with 162 additions and 61 deletions

95
package-lock.json generated
View File

@ -685,6 +685,74 @@
"chalk": "^3.0.0" "chalk": "^3.0.0"
} }
}, },
"@oozcitak/dom": {
"version": "1.15.5",
"resolved": "https://registry.npmjs.org/@oozcitak/dom/-/dom-1.15.5.tgz",
"integrity": "sha512-L6v3Mwb0TaYBYgeYlIeBaHnc+2ZEaDSbFiRm5KmqZQSoBlbPlf+l6aIH/sD5GUf2MYwULw00LT7+dOnEuAEC0A==",
"requires": {
"@oozcitak/infra": "1.0.5",
"@oozcitak/url": "1.0.0",
"@oozcitak/util": "8.0.0"
},
"dependencies": {
"@oozcitak/util": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.0.0.tgz",
"integrity": "sha512-+9Hq6yuoq/3TRV/n/xcpydGBq2qN2/DEDMqNTG7rm95K6ZE2/YY/sPyx62+1n8QsE9O26e5M1URlXsk+AnN9Jw=="
}
}
},
"@oozcitak/infra": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@oozcitak/infra/-/infra-1.0.5.tgz",
"integrity": "sha512-o+zZH7M6l5e3FaAWy3ojaPIVN5eusaYPrKm6MZQt0DKNdgXa2wDYExjpP0t/zx+GoQgQKzLu7cfD8rHCLt8JrQ==",
"requires": {
"@oozcitak/util": "8.0.0"
},
"dependencies": {
"@oozcitak/util": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.0.0.tgz",
"integrity": "sha512-+9Hq6yuoq/3TRV/n/xcpydGBq2qN2/DEDMqNTG7rm95K6ZE2/YY/sPyx62+1n8QsE9O26e5M1URlXsk+AnN9Jw=="
}
}
},
"@oozcitak/url": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@oozcitak/url/-/url-1.0.0.tgz",
"integrity": "sha512-LGrMeSxeLzsdaitxq3ZmBRVOrlRRQIgNNci6L0VRnOKlJFuRIkNm4B+BObXPCJA6JT5bEJtrrwjn30jueHJYZQ==",
"requires": {
"@oozcitak/infra": "1.0.3",
"@oozcitak/util": "1.0.2"
},
"dependencies": {
"@oozcitak/infra": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@oozcitak/infra/-/infra-1.0.3.tgz",
"integrity": "sha512-9O2wxXGnRzy76O1XUxESxDGsXT5kzETJPvYbreO4mv6bqe1+YSuux2cZTagjJ/T4UfEwFJz5ixanOqB0QgYAag==",
"requires": {
"@oozcitak/util": "1.0.1"
},
"dependencies": {
"@oozcitak/util": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-1.0.1.tgz",
"integrity": "sha512-dFwFqcKrQnJ2SapOmRD1nQWEZUtbtIy9Y6TyJquzsalWNJsKIPxmTI0KG6Ypyl8j7v89L2wixH9fQDNrF78hKg=="
}
}
},
"@oozcitak/util": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-1.0.2.tgz",
"integrity": "sha512-4n8B1cWlJleSOSba5gxsMcN4tO8KkkcvXhNWW+ADqvq9Xj+Lrl9uCa90GRpjekqQJyt84aUX015DG81LFpZYXA=="
}
}
},
"@oozcitak/util": {
"version": "8.3.3",
"resolved": "https://registry.npmjs.org/@oozcitak/util/-/util-8.3.3.tgz",
"integrity": "sha512-Ufpab7G5PfnEhQyy5kDg9C8ltWJjsVT1P/IYqacjstaqydG4Q21HAT2HUZQYBrC/a1ZLKCz87pfydlDvv8y97w=="
},
"@opencensus/core": { "@opencensus/core": {
"version": "0.0.9", "version": "0.0.9",
"resolved": "https://registry.npmjs.org/@opencensus/core/-/core-0.0.9.tgz", "resolved": "https://registry.npmjs.org/@opencensus/core/-/core-0.0.9.tgz",
@ -1170,15 +1238,6 @@
"integrity": "sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==", "integrity": "sha512-l42BggppR6zLmpfU6fq9HEa2oGPEI8yrSPL3GITjfRInppYFahObbIQOQK3UGxEnyQpltZLaPe75046NOZQikw==",
"dev": true "dev": true
}, },
"@types/xml2js": {
"version": "0.4.5",
"resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.5.tgz",
"integrity": "sha512-yohU3zMn0fkhlape1nxXG2bLEGZRc1FeqF80RoHaYXJN7uibaauXfhzhOJr1Xh36sn+/tx21QAOf07b/xYVk1w==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/yargs": { "@types/yargs": {
"version": "15.0.4", "version": "15.0.4",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.4.tgz", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.4.tgz",
@ -8199,20 +8258,16 @@
"integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==",
"dev": true "dev": true
}, },
"xml2js": { "xmlbuilder2": {
"version": "0.4.23", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", "resolved": "https://registry.npmjs.org/xmlbuilder2/-/xmlbuilder2-2.1.0.tgz",
"integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", "integrity": "sha512-hWCfRTKFhhLv1QlSRn+PMwu0knJUQ5PSQanHFoY9Zy1q8cMEm0/37PHbeiBwlplgNafDzDJRdwQkMYVdXt77Nw==",
"requires": { "requires": {
"sax": ">=0.6.0", "@oozcitak/dom": "1.15.5",
"xmlbuilder": "~11.0.0" "@oozcitak/infra": "1.0.5",
"@oozcitak/util": "8.3.3"
} }
}, },
"xmlbuilder": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
"integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
},
"xmlchars": { "xmlchars": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",

View File

@ -12,7 +12,7 @@
"react": "^16.13.0", "react": "^16.13.0",
"react-dom": "^16.13.0", "react-dom": "^16.13.0",
"smtp-server": "^3.6.0", "smtp-server": "^3.6.0",
"xml2js": "^0.4.23" "xmlbuilder2": "^2.1.0"
}, },
"devDependencies": { "devDependencies": {
"@types/express": "^4.17.3", "@types/express": "^4.17.3",
@ -24,7 +24,6 @@
"@types/react": "^16.9.29", "@types/react": "^16.9.29",
"@types/react-dom": "^16.9.5", "@types/react-dom": "^16.9.5",
"@types/smtp-server": "^3.5.4", "@types/smtp-server": "^3.5.4",
"@types/xml2js": "^0.4.5",
"axios": "^0.19.2", "axios": "^0.19.2",
"concurrently": "^5.1.0", "concurrently": "^5.1.0",
"jest": "^25.2.4", "jest": "^25.2.4",

View File

@ -1,21 +1,19 @@
import xml2js from "xml2js"; import * as xmlbuilder2 from "xmlbuilder2";
import fs from "fs"; import fs from "fs";
(async () => { for (const feed of fs
for (const feed of fs
.readdirSync("static/feeds") .readdirSync("static/feeds")
.filter(file => !file.startsWith("."))) { .filter(file => !file.startsWith("."))) {
try { try {
const xml = await new xml2js.Parser().parseStringPromise( const xml: any = xmlbuilder2.convert(
fs.readFileSync(`static/feeds/${feed}`, "utf8") fs.readFileSync(`static/feeds/${feed}`, "utf8"),
{ format: "object", wellFormed: true }
); );
if (xml?.feed?.updated === undefined) if (xml?.feed?.updated === undefined)
throw new Error("Cant find xml.feed.updated"); throw new Error("Cant find xml.feed.updated");
new xml2js.Builder().buildObject(xml);
console.log(`OK ${feed}`); console.log(`OK ${feed}`);
} catch (error) { } catch (error) {
console.log(`ERROR ${feed}: ${error}`); console.log(`ERROR ${feed}: ${error}`);
} }
} }
console.log("FINISHED"); console.log("FINISHED");
})();

View File

@ -3,7 +3,7 @@ import { SMTPServer } from "smtp-server";
import mailparser from "mailparser"; import mailparser from "mailparser";
import React from "react"; import React from "react";
import ReactDOMServer from "react-dom/server"; import ReactDOMServer from "react-dom/server";
import xml2js from "xml2js"; import * as xmlbuilder2 from "xmlbuilder2";
import { promises as fs } from "fs"; import { promises as fs } from "fs";
import cryptoRandomString from "crypto-random-string"; import cryptoRandomString from "crypto-random-string";
@ -82,9 +82,10 @@ export const emailServer = new SMTPServer({
const path = feedPath(identifier); const path = feedPath(identifier);
const xmlText = await fs.readFile(path, "utf8").catch(() => null); const xmlText = await fs.readFile(path, "utf8").catch(() => null);
if (xmlText === null) continue; if (xmlText === null) continue;
const xml = await new xml2js.Parser().parseStringPromise(xmlText); const xml = parseXML(xmlText);
xml.feed.updated = now(); xml.feed.updated = now();
if (xml.feed.entry === undefined) xml.feed.entry = []; if (xml.feed.entry === undefined) xml.feed.entry = [];
if (!Array.isArray(xml.feed.entry)) xml.feed.entry = [xml.feed.entry];
xml.feed.entry.unshift(entry); xml.feed.entry.unshift(entry);
while (xml.feed.entry.length > 0 && renderXML(xml).length > 500_000) while (xml.feed.entry.length > 0 && renderXML(xml).length > 500_000)
xml.feed.entry.pop(); xml.feed.entry.pop();
@ -94,7 +95,7 @@ export const emailServer = new SMTPServer({
})().catch(error => { })().catch(error => {
console.error(error); console.error(error);
stream.resume(); stream.resume();
callback(error); callback(new Error("Failed to receive message. Please try again."));
}); });
} }
}).listen(process.env.EMAIL_PORT ?? 2525); }).listen(process.env.EMAIL_PORT ?? 2525);
@ -207,21 +208,17 @@ function Created({ identifier }: { identifier: string }) {
function Feed({ name, identifier }: { name: string; identifier: string }) { function Feed({ name, identifier }: { name: string; identifier: string }) {
return { return {
feed: { feed: {
$: { xmlns: "http://www.w3.org/2005/Atom" }, "@xmlns": "http://www.w3.org/2005/Atom",
link: [ link: [
{ {
$: { "@rel": "self",
rel: "self", "@type": "application/atom+xml",
type: "application/atom+xml", "@href": feedURL(identifier)
href: feedURL(identifier)
}
}, },
{ {
$: { "@rel": "alternate",
rel: "alternate", "@type": "text/html",
type: "text/html", "@href": "https://www.kill-the-newsletter.com/"
href: "https://www.kill-the-newsletter.com/"
}
} }
], ],
id: urn(identifier), id: urn(identifier),
@ -258,13 +255,11 @@ function Entry({
author: { name: author }, author: { name: author },
updated: now(), updated: now(),
link: { link: {
$: { "@rel": "alternate",
rel: "alternate", "@type": "text/html",
type: "text/html", "@href": "https://www.kill-the-newsletter.com/entry"
href: "https://www.kill-the-newsletter.com/entry"
}
}, },
content: { $: { type: "html" }, _: content } content: { "@type": "html", "#": content }
} }
}; };
} }
@ -301,5 +296,18 @@ function renderHTML(component: React.ReactElement): string {
} }
function renderXML(xml: object): string { function renderXML(xml: object): string {
return new xml2js.Builder().buildObject(xml); return xmlbuilder2.convert({ invalidCharReplacement: "" }, xml, {
format: "xml",
wellFormed: true,
noDoubleEncoding: true,
prettyPrint: true
});
}
function parseXML(xml: string): any {
return xmlbuilder2.convert({ invalidCharReplacement: "" }, xml, {
format: "object",
wellFormed: true,
noDoubleEncoding: true
});
} }

View File

@ -11,6 +11,21 @@ test("create feed", async () => {
}); });
describe("receive email", () => { describe("receive email", () => {
test("updated field is updated", async () => {
const identifier = await createFeed();
const before = await getFeed(identifier);
await emailClient.sendMail({
from: "publisher@example.com",
to: `${identifier}@kill-the-newsletter.com`,
subject: "New Message",
html: "<p>HTML content</p>"
});
const after = await getFeed(identifier);
expect(after.match(/<updated>(.*)<\/updated>/)![1]).not.toMatch(
before.match(/<updated>(.*)<\/updated>/)![1]
);
});
test("HTML content", async () => { test("HTML content", async () => {
const identifier = await createFeed(); const identifier = await createFeed();
await emailClient.sendMail({ await emailClient.sendMail({
@ -50,6 +65,32 @@ describe("receive email", () => {
expect(feed).toMatch(`href="https://www.kill-the-newsletter.com"`); expect(feed).toMatch(`href="https://www.kill-the-newsletter.com"`);
}); });
test("invalid XML character in HTML", async () => {
const identifier = await createFeed();
await emailClient.sendMail({
from: "publisher@example.com",
to: `${identifier}@kill-the-newsletter.com`,
subject: "New Message",
html: "<p>Invalid XML character (backspace): \b</p>"
});
const feed = await getFeed(identifier);
expect(feed).toMatch("Invalid XML character (backspace): ");
});
test("invalid XML character in text", async () => {
const identifier = await createFeed();
await emailClient.sendMail({
from: "publisher@example.com",
to: `${identifier}@kill-the-newsletter.com`,
subject: "New Message",
text: "Invalid XML character (backspace): \b"
});
const feed = await getFeed(identifier);
expect(feed).toMatch(
"Invalid XML character (backspace): &lsquo;&#x8;&rsquo;"
);
});
test("missing content", async () => { test("missing content", async () => {
const identifier = await createFeed(); const identifier = await createFeed();
await emailClient.sendMail({ await emailClient.sendMail({