Generate newsletter test pages in a subdirectory

packages/readabilityjs/test/generate-testcase.js

@@ -30,6 +30,11 @@ const enableJavascriptForUrl = (url) => {
 };
 
 function generateTestcase(slug) {
+  const options = {};
+  if (slug.startsWith("newsletters/")) {
+    // keep the newsletter content in tables
+    options.keepTables = true;
+  }
   var destRoot = path.join(testcaseRoot, slug);
 
   fs.mkdir(destRoot, function (err) {
@@ -42,12 +47,12 @@ function generateTestcase(slug) {
               console.error("Source existed but couldn't be read?");
               process.exit(1);
             }
-            onResponseReceived(null, data, destRoot);
+            onResponseReceived(null, data, destRoot, options);
           });
         } else {
           fs.writeFile(path.join(destRoot, 'url.txt'), argURL, () => null);
           fetchSource(argURL, function (fetchErr, data) {
-            onResponseReceived(fetchErr, data, destRoot);
+            onResponseReceived(fetchErr, data, destRoot, options);
           });
         }
       });
@@ -55,7 +60,7 @@ function generateTestcase(slug) {
     }
     fs.writeFile(path.join(destRoot, 'url.txt'), argURL, () => null);
     fetchSource(argURL, function (fetchErr, data) {
-      onResponseReceived(fetchErr, data, destRoot);
+      onResponseReceived(fetchErr, data, destRoot, options);
     });
   });
 }
@@ -198,7 +203,7 @@ function sanitizeSource(html, callbackFn) {
   }, callbackFn);
 }
 
-function onResponseReceived(error, source, destRoot) {
+function onResponseReceived(error, source, destRoot, options) {
   if (error) {
     console.error("Couldn't tidy source html!");
     console.error(error);
@@ -217,11 +222,11 @@ function onResponseReceived(error, source, destRoot) {
     if (debug) {
       console.log("Running readability stuff");
     }
-    await runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json"));
+    await runReadability(source, path.join(destRoot, "expected.html"), path.join(destRoot, "expected-metadata.json"), options);
   });
 }
 
-async function runReadability(source, destPath, metadataDestPath) {
+async function runReadability(source, destPath, metadataDestPath, options) {
   var uri = "http://fakehost/test/page.html";
   var myReader, result, readerable;
   try {
@@ -230,7 +235,7 @@ async function runReadability(source, destPath, metadataDestPath) {
     readerable = isProbablyReaderable(dom);
     // We pass `caption` as a class to check that passing in extra classes works,
     // given that it appears in some of the test documents.
-    myReader = new Readability(dom, { classesToPreserve: ["caption"], url: uri });
+    myReader = new Readability(dom, { classesToPreserve: ["caption"], url: uri, ...options });
     result = await myReader.parse();
   } catch (ex) {
     console.error(ex);

packages/readabilityjs/test/utils.js

@@ -14,14 +14,23 @@ var testPageRoot = path.join(__dirname, "test-pages");
 
 exports.getTestPages = function(isOmnivore = null) {
   const root = isOmnivore ? `${testPageRoot}/omnivore` : testPageRoot;
-  return fs.readdirSync(root).filter(dir => dir !== 'omnivore').map(function(dir) {
-    return {
+  const testPages = [];
+  const testPageDirs = fs.readdirSync(root).filter(dir => dir !== 'omnivore');
+  testPageDirs.forEach(function(dir) {
+    if (dir === 'newsletters') {
+      // newsletters are a special case, they are in a subdirectory
+      testPageDirs.push(fs.readdirSync(path.join(root, dir)).map(subdir => path.join(dir, subdir)));
+      return;
+    }
+
+    testPages.push({
       dir: dir,
       source: readFile(path.join(root, dir, "source.html")),
       expectedContent: readFile(path.join(root, dir, "expected.html")),
       expectedMetadata: readJSON(path.join(root, dir, "expected-metadata.json")),
-    };
+    });
   });
+  return testPages;
 };
 
 exports.prettyPrint = function(html) {