Automated Lighthouse Reporting using Puppeteer

bash
npm init # Create your package.json file
npm install --save puppeteer lighthouse express levelup leveldown
npm install --save-dev esm$$$

bash
touch lighthouse-util.js
touch service.js
touch reports.js
touch index.html
touch playground.js$$$$$

js
import puppeteer from "puppeteer";

export function createBrowser() {
   return puppeteer.launch({
     args: ["--show-paint-rects"] // Required by lighthouse
   });
}

js
import lighthouse from "lighthouse"; // This should be at the top of the file

export function createReportWithBrowser(browser, url, options = { output: "html" }) {
  const endpoint = browser.wsEndpoint(); // Allows us to talk via DevTools protocol
  const endpointURL = new URL(endpoint); // Lighthouse only cares about the port, so we have to parse the URL so we can grab the port to talk to Chrome on
  return lighthouse(
    url,
    Object.assign({}, {
      port: endpointURL.port
    }, options) // Allow options to override anything here
  );
}

js
import { createBrowser, createReportWithBrowser } from "./lighthouse-util.js";
import fs from "fs";
import Assert from "assert";

// IIFE (https://developer.mozilla.org/en-US/docs/Glossary/IIFE) so that we can use async in the top level
(async () => {
  
  const browser = await createBrowser();
  
  const result = await createReportWithBrowser(
    browser,
    "https://example.com",
    {
        output: "html"  
    }
  );

  Assert(result.report, "No report returned");

  fs.writeFileSync("report.html", result.report, "utf-8");
  
  await browser.close();
})()
    // Catch anything that went wrong!
    .catch(console.error)
    .then(() => {
       console.log("Finished!");
    });

bash
node -r esm playground.js$

json

js
import levelup from "levelup";
import leveldown from "leveldown";

export function createReportStore() {
  const database = leveldown("./store");
  return levelup(database);
}

bash
npm install --save level-jobs$

js
 // These should be at the top of the file
import assert from "assert";
import { createBrowser, createReportWithBrowser } from "./lighthouse-util.js";

async function doReportWork(store, payload) {
  assert(payload.id, "Expected payload to have an id");
  assert(payload.url, "Expected payload to have a url");
  
     const browser = await createBrowser();  
  
  const result = await createReportWithBrowser(
    browser,
    payload.url,
    payload.options || { output: "html" }
  );

  await browser.close();

  // Save our result ready to be retrieved by the client
  console.log(`Saving report for ${payload.id}`);

  const document = Object.assign({}, payload, {
    result
  });

  await store.put(payload.id, JSON.stringify(document)); 
}

function createReportWorker(store) {
  return (unused, payload, callback) => {
    doReportWork(store, payload)
        .then(
          () => callback(),
          (error) => callback(error)
      );
  };
}

js
import Jobs from "level-jobs"; // This should be at the top of the file

export function createReportQueue(store) {
  const options = {
        maxConcurrency: 1
  };
  return Jobs(store, createReportWorker(store), options);
}

bash
npm install --save uuid$

js
import uuid from "uuid"; // This should be at the top of the file

export async function requestGenerateReport(store, queue, url, options = { output: "html" }) {
  const id = `report:${uuid.v4()}`;
  // Notice the use of JSON.stringify, levelup will accept Buffers or strings, so we want
  // to use JSON for our value
  const document = {
    id,
    url,
    options
  };
  await store.put(id, JSON.stringify(document));
  await new Promise(
    (resolve, reject) => queue.push(document, error => error ? reject(error) : resolve())
  );
  return id;
}

js
import express from "express";
import { createReportQueue, createReportStore, requestGenerateReport } from "./reports.js";
import assert from "assert";

const app = express();

// Allows us to grab these within our request handlers
app.locals.store = createReportStore();
app.locals.queue = createReportQueue(app.locals.store);

app.post("/report", express.json(), (request, response, next) => {
  if (!Array.isArray(request.body)) {
    return response.sendStatus(400); // Bad request, we expected an array
  }
  Promise.all(
      request.body.map(async ({ url, options }) => {
      assert(typeof url === "string", "Expected url to be provided");
         // We want to return an array for each item so we can match up the url
      return [
        url,
          await requestGenerateReport(
          request.app.locals.store,
          request.app.locals.queue,
          url,
          options
        ) 
      ];
    })
  )
      .then(identifiers => response.send(identifiers))
      // Catch any errors and allow express to handle it
      .catch(next);
});

js
app.get("/report/:id", (request, response, next) => {
  request.app.locals.store
      .get(request.params.id)
        .then(report => {
        if (!report) {
          return response.sendStatus(404); // We couldn't find it
      }
        return response.send(report);
      })
      // Catch any errors and allow express to handle it
      .catch(next);
});

js
app.delete("/report/:id", (request, response, next) => {
    request.app.locals.store
      .get(request.params.id)
        .then(report => {
        if (!report) {
          return response.sendStatus(404); // We couldn't find it, may have already been deleted
      }
      return app.locals.store
          .del(request.params.id)
          .then(() => response.sendStatus(204)); // All deleted
      })
      // Catch any errors and allow express to handle it
      .catch(next);
});

js
app.get("/", (request, response) => response.sendFile(`${process.cwd()}/index.html`));

js
// IIFE so we don't need to define `port` as `let` ¯\_(ツ)_/¯
const port = (() => {
  if (/^\d+$/.test(process.env.PORT)) {
      return +process.env.PORT;  
  }
  // Maybe you have other defaults you want to check here to decide on the port
  return 8080;
})();

js
app.listen(port, () => console.log(`Listening on port ${port}`));

html
<html>
<body>
<textarea id="urls" rows="10"></textarea>
<button id="schedule" type="button">Schedule Reports</button>
<ul id="results"></ul>
<script type="application/javascript">
/* Our code will code here */
</script>
</html>

js
const urls = document.querySelector("#urls"),
      schedule = document.querySelector("#schedule"),
      results = document.querySelector("#results");

js
function processUrls(value) {
     if (!value) {
        return { message: "No urls provided" }; 
  }
  
  const validatedUrls = value
      .split("\n") // Split by line
      .reduce((urls, split) => urls.concat(split.split(",")), []) // Split by comma
      .map(url => url.trim()) // Trim each value, so we don't have extra white space
      .map(url => {
             // Creating a URL will do the "validation" for us, it will throw if it is invalid
            try {
        const instance = new URL(url);
        if (!instance.origin || !instance.protocol) {
          // We want to have an origin or protocol so that 
          // we can reference it correctly in our service
            return false;
        }
                return instance.toString();
      } catch(e) {
        // Return false if any errors so we can alert the user of an issue
           return false; 
      }
    });
  
    // So we can visualise whats happening in our dev tools
    console.log({ validatedUrls });
  
  if (validatedUrls.includes(false)) {
      return { message: "One or more url provided is not valid" };
  } 
  
  return { validUrls: validatedUrls };
}

js
function deleteExternalReport(identifier) {
  fetch(`/report/${identifier}`, {
        method: "DELETE"
  })
      .catch(console.warn)
}

js
function checkResults(element, identifier, originalUrl) {
  const intervalHandle = setInterval(doCheck, 2500);
  function doCheck() {
    fetch(`/report/${identifier}`)
        .then(response => response.json())
        .then(report => {
          if (!(report.result && report.result.report)) {
            return;
        }
          // We don't need to check again
          clearInterval(intervalHandle);
          // Create a data uri that can be opened directly
          const link = document.createElement("a");
          link.innerText = `Complete: ${originalUrl}`;
          // Open up in a new tab
          link.target = "_blank";
          link.href = URL.createObjectURL(new Blob([report.result.report], { type: "text/html" }));
          // Empty out the element
          element.innerHTML = ``;
          element.appendChild(link);
                deleteExternalReport(identifier);
        })
        .catch(console.warn)
  }
}

js
function displayResults(identifiers) {
    identifiers.forEach(
        // Each item will be an array with the original url and the new identifier
    ([originalUrl, identifier]) => {
        const element = document.createElement("li");
            element.innerText = `Processing: ${originalUrl}`;
            // Append to our results list
      results.appendChild(element);
            checkResults(element, identifier, originalUrl);
    }
  )
}

js
function send(urls) {
     // We want to receive an html output
  const options = {
      output: "html" 
  };
  fetch("/report", {
        method: "POST",
    headers: {
        "Content-Type": "application/json" 
    },
    body: JSON.stringify(urls.map(url => ({ url })))
  })
      .then(response => response.json())
      .then(displayResults)
      .catch(() => alert("Something went wrong while sending our request"))
}

js
schedule.addEventListener("click", () => {
  const value = urls.value;
  
  const { validUrls, message } = processUrls(value);
  
  if (message) {
      return alert(message); 
  }
  
  // Clear out the textarea so we can schedule more urls
  urls.value = "";
  
  // Trigger the process
  send(validUrls);
});