Add a pre-commit hook to lint posts that are being modified

.github/workflows/auto-compress-images.yaml

@@ -1,4 +1,6 @@
 name: Compress Images Once a Month - Creates Pull Request
+env:
+  HUSKY: 0
 
 on:
   schedule:

.github/workflows/call-algolia-deployment-script.yml

@@ -1,5 +1,7 @@
 # .github/workflows/call-algolia-deployment-script.yml
 name: Call algolia deployment script on merge
+env:
+  HUSKY: 0
 
 on:
   push:

.github/workflows/check-a11y-of-changed-content.yaml

@@ -1,4 +1,6 @@
 name: Check accessibility of changed content
+env:
+  HUSKY: 0
 
 on:
   pull_request:

.github/workflows/generate-related.yaml

@@ -1,5 +1,7 @@
 name: Generate Read More related
-
+env:
+  HUSKY: 0
+
 on:
   workflow_dispatch:
     branches:

.github/workflows/lint.yaml

@@ -1,4 +1,6 @@
 name: Lint Posts
+env:
+  HUSKY: 0
 
 on:
   pull_request:

.github/workflows/remove-unused-images.yaml

@@ -1,4 +1,6 @@
 name: Remove Unused Images
+env:
+  HUSKY: 0
 
 on:
   workflow_dispatch:

.husky/pre-push

@@ -0,0 +1 @@
+npm run lint-on-push

README.md

@@ -184,6 +184,10 @@ Then you can navigate to [localhost][localhost] in your browser.
 Note that if you performed a _sparse checkout_ as recommended, and if this is your first post, then you won't see any
 blog posts when the site loads unless you've already added a file for your new blog post.
 
+### Pushing to GitHub
+
+We now have a pre-push hook defined that will perform linting against changed blog posts before any push command goes ahead. This should ensure that errors in the metadata for posts are caught before they are built, as it can be much harder to determine why your post is not appearing from the pages-build-deployment GitHub action logs. If you have run `npm install` then it should automatically take care of setting up the hooks using [Husky](https://typicode.github.io/husky/). If for any reason this is blocking you from pushing and you really need to, you can skip the hook by running `git push --no-verify`.
+
 ## CI/CD
 
 We use GitHub Actions for CI/CD. The workflow definitions are in YAML files

lintHelper.js

@@ -0,0 +1,100 @@
+const matter = require("gray-matter");
+const yaml = require("js-yaml");
+const fs = require("fs");
+const clc = require("cli-color");
+
+const MAX_CATEGORIES = 3;
+
+const errorColour = clc.red.bold;
+const warningColour = clc.yellow;
+
+const logError = (...params) =>
+  console.error(errorColour(...params));
+
+const logWarning = (...params) =>
+  console.warn(warningColour(...params));
+
+const flatMap = (arr, mapFunc) =>
+  arr.reduce((prev, x) => prev.concat(mapFunc(x)), []);
+
+const getValidCategories = () => {
+    const categoriesYaml = yaml.safeLoad(
+      fs.readFileSync("_data/categories.yml", "utf8")
+    );
+
+    const categories = flatMap(
+      // remove 'Latest Articles' which is a pseudo-category
+      categoriesYaml.filter(c => c.url.startsWith("/category/")),
+      // merge category title into sub-categories
+      c => [c.title].concat(c.subcategories ? c.subcategories : [])
+    ).map(c => c.toLowerCase());
+
+    console.log("Valid categories are: " + categories.join(', '));
+
+    return categories;
+};
+
+const lintPost = (path, categories) => {
+      try {
+        const blogPost = fs.readFileSync(path, "utf8");
+        const frontMatter = matter(blogPost);
+        const frontMatterCats = frontMatter.data.categories;
+
+        let category;
+        let postCategories;
+        // if the frontmatter defines a 'category' field:
+        if (frontMatter.data.category) {
+          category = frontMatter.data.category.toLowerCase();
+          postCategories = [category];
+        // if the frontmatter defines a 'categories' field with at least one but no more than 3 values:
+
+        } else if (frontMatterCats && frontMatterCats.length && frontMatterCats.length <= MAX_CATEGORIES) {
+          postCategories = frontMatter.data.categories.map(c => c.toLowerCase());
+          category = postCategories[0];
+        } else {
+          logError("The post " + path + " does not have at least one and no more than " + MAX_CATEGORIES + " categories defined.");
+          return false;
+        }
+
+        if (!categories.includes(category)) {
+          logError(
+            "The post " + path + " does not have a recognised category"
+          );
+          return false;
+        } else {
+          postCategories
+            .filter(c => !categories.includes(c))
+            .forEach(c => logWarning(
+              "The post " + path + " has an unrecognised category: '" + c + "'. Check spelling or remove/move to tags."
+            ));
+        }
+
+        const summary = frontMatter.data.summary;
+        const pathArray = path.split("/");
+        const postDateString = pathArray[pathArray.length - 1].substring(0, 10);
+        const postDate = new Date(postDateString);
+        if (postDate > new Date("2018-03-26")) {
+          // Note _prose.yml specifies 130 characters are needed, so if you change this please also change the instructions
+          if(!summary) {
+              logError("The post " + path + " does not have a summary.")
+              return false;
+          }
+          else if (summary.length < 130) {
+            logWarning(
+              "The post " + path + " summary length is " + summary.length + ". Recommended minimum length for the summary is 130 characters."
+            );
+          }
+        }
+      } catch (e) {
+        logError(path, e);
+        return false;
+      }
+      return true;
+    }
+
+    module.exports = {
+        logError,
+        logWarning,
+        getValidCategories,
+        lintPost
+    }

lintOnPush.js

@@ -0,0 +1,76 @@
+const { execSync } = require("child_process");
+const { logError, lintPost, getValidCategories } = require("./lintHelper");
+const readline = require("readline")
+
+const LINTER_MATCH_PATTERN = /^_posts.*\.(md|markdown|html)$/;
+const EMPTY_OBJECT_PATTERN = /^0+$/;
+
+const getChangedFiles = async () => {
+    const rl = readline.createInterface({input: process.stdin, crlfDelay: Infinity});
+
+    const ranges = [];
+
+    for await (const line of rl) {
+        const [_localRef, localSha, _remoteRef, remoteSha] = line.trim().split(/\s+/);
+
+        if (!localSha || localSha.match(EMPTY_OBJECT_PATTERN)) {
+            continue; // branch deleted, ignore
+        }
+
+        if (!remoteSha || remoteSha.match(EMPTY_OBJECT_PATTERN)) {
+            // new branch
+            ranges.push(localSha);
+        } else {
+            ranges.push(`${remoteSha}..${localSha}`);
+        }
+    }
+    if (ranges.length === 0) {
+        return [];
+    }
+
+    let files = [];
+    for (const range of ranges) {
+        try {
+            const diffOutput = execSync(
+                `git diff --name-only ${range}`,
+                { encoding: "utf8" }
+            );
+            files.push(...diffOutput.split("\n").filter(file => file.match(LINTER_MATCH_PATTERN)));
+        } catch {
+            // ignore empty diffs
+        }
+    }
+
+    // Ensure unique list
+    return [...new Set(files)];
+}
+
+const lintOnPush = async () => {
+    const changedFiles = await getChangedFiles();
+
+    if (changedFiles.length === 0) {
+        console.log("No relevant post files changed.");
+        process.exit(0);
+    }
+
+    console.log("Linting posts to be pushed:", changedFiles);
+
+    let fail = false;
+    const categories = getValidCategories();
+    for (const file of changedFiles) {
+        if (!lintPost(file, categories))
+        {
+            fail = true;
+        }
+    }
+
+    if (fail) {
+        logError("Push blocked due to linting errors.");
+        process.exit(1);
+    }
+}
+
+lintOnPush().catch((err) => {
+  console.error("Unexpected error in pre-push hook:", err);
+  process.exit(1);
+});

lintPosts.js

@@ -1,22 +1,9 @@
 const globby = require("globby");
-const matter = require("gray-matter");
 const yaml = require("js-yaml");
 const fs = require("fs");
-const clc = require("cli-color");
-const LINTER_MATCH_PATTERN="_posts/**/*.{md,markdown,html}";
-const MAX_CATEGORIES = 3;
-
-const errorColour = clc.red.bold;
-const warningColour = clc.yellow;
-
-const logError = (...params) =>
-  console.error(errorColour(...params));
+const { logError, getValidCategories, lintPost } = require("./lintHelper");
 
-const logWarning = (...params) =>
-  console.warn(warningColour(...params));
-
-const flatMap = (arr, mapFunc) =>
-  arr.reduce((prev, x) => prev.concat(mapFunc(x)), []);
+const LINTER_MATCH_PATTERN="_posts/**/*.{md,markdown,html}";
 
 const lintAuthorsYml = () => {
   const authorsPath = "_data/authors.yml";
@@ -55,78 +42,15 @@ const lintAuthorsYml = () => {
 };
 
 const lintPosts = () => {
-  const categoriesYaml = yaml.safeLoad(
-    fs.readFileSync("_data/categories.yml", "utf8")
-  );
-
-  const categories = flatMap(
-    // remove 'Latest Articles' which is a pseudo-category
-    categoriesYaml.filter(c => c.url.startsWith("/category/")),
-    // merge category title into sub-categories
-    c => [c.title].concat(c.subcategories ? c.subcategories : [])
-  ).map(c => c.toLowerCase());
-
-  console.log("Valid categories are: " + categories.join(', '));
+  const categories = getValidCategories();
 
   let fail = false;
 
   // lint each blog post
   globby([LINTER_MATCH_PATTERN]).then(paths => {
     paths.forEach(path => {
-      try {
-        const blogPost = fs.readFileSync(path, "utf8");
-        const frontMatter = matter(blogPost);
-        const frontMatterCats = frontMatter.data.categories;
-
-        let category;
-        let postCategories;
-        // if the frontmatter defines a 'category' field:
-        if (frontMatter.data.category) {
-          category = frontMatter.data.category.toLowerCase();
-          postCategories = [category];
-        // if the frontmatter defines a 'categories' field with at least one but no more than 3 values:
-
-        } else if (frontMatterCats && frontMatterCats.length && frontMatterCats.length <= MAX_CATEGORIES) {
-          postCategories = frontMatter.data.categories.map(c => c.toLowerCase());
-          category = postCategories[0];
-        } else {
-          logError("The post " + path + " does not have at least one and no more than " + MAX_CATEGORIES + " categories defined.");
-          fail = true;
-          return;
-        }
-
-        if (!categories.includes(category)) {
-          logError(
-            "The post " + path + " does not have a recognised category"
-          );
-          fail = true;
-        } else {
-          postCategories
-            .filter(c => !categories.includes(c))
-            .forEach(c => logWarning(
-              "The post " + path + " has an unrecognised category: '" + c + "'. Check spelling or remove/move to tags."
-            ));
-        }
-
-
-        const summary = frontMatter.data.summary;
-        const pathArray = path.split("/");
-        const postDateString = pathArray[pathArray.length - 1].substring(0, 10);
-        const postDate = new Date(postDateString);
-        if (postDate > new Date("2018-03-26")) {
-          // Note _prose.yml specifies 130 characters are needed, so if you change this please also change the instructions
-          if(!summary) {
-              logError("The post " + path + " does not have a summary.")
-              fail = true;
-          }
-          else if (summary.length < 130) {
-            logWarning(
-              "The post " + path + " summary length is " + summary.length + ". Recommended minimum length for the summary is 130 characters."
-            );
-          }
-        }
-      } catch (e) {
-        logError(path, e);
+      if (!lintPost(path, categories))
+      {
         fail = true;
       }
     });

package.json

@@ -14,6 +14,7 @@
     "glob-promise": "^4.2.2",
     "globby": "^7.1.1",
     "gray-matter": "^3.1.1",
+    "husky": "^9.1.7",
     "js-yaml": "^3.10.0",
     "markdown-spellcheck": "^1.3.1",
     "markdown-to-txt": "^2.0.0",
@@ -32,12 +33,14 @@
   },
   "scripts": {
     "lint": "node lintPosts.js",
+    "lint-on-push": "node lintOnPush.js",
     "compute-embeddings": "node scripts/generate-related/compute-embeddings.js",
     "generate-related": "node scripts/generate-related/blog-metadata.js",
     "remove-unused-images": "node scripts/images/remove-images.js",
     "spellcheck": "mdspell \"**/ceberhardt/_posts/*.md\" --en-gb -a -n -x -t",
     "style": "sass --no-source-map --style=compressed scss/style.scss style.css",
-    "scripts": "uglifyjs scripts/initialise-menu.js scripts/jquery-1.9.1.js scripts/jquery.jscroll-2.2.4.js scripts/load-clap-count.js scripts/elapsed.js scripts/graft-studio/header-scroll.js scripts/graft-studio/jquery.mmenu.all.js scripts/graft-studio/jquery.matchHeight.js node_modules/applause-button/dist/applause-button.js node_modules/cookieconsent/build/cookieconsent.min.js -o script.js"
+    "scripts": "uglifyjs scripts/initialise-menu.js scripts/jquery-1.9.1.js scripts/jquery.jscroll-2.2.4.js scripts/load-clap-count.js scripts/elapsed.js scripts/graft-studio/header-scroll.js scripts/graft-studio/jquery.mmenu.all.js scripts/graft-studio/jquery.matchHeight.js node_modules/applause-button/dist/applause-button.js node_modules/cookieconsent/build/cookieconsent.min.js -o script.js",
+    "prepare": "husky || true"
   },
   "homepage": "http://blog.scottlogic.com",
   "private": true