diff --git a/.github/workflows/auto-compress-images.yaml b/.github/workflows/auto-compress-images.yaml index 4c0e74b12f..c827e28056 100644 --- a/.github/workflows/auto-compress-images.yaml +++ b/.github/workflows/auto-compress-images.yaml @@ -1,4 +1,6 @@ name: Compress Images Once a Month - Creates Pull Request +env: + HUSKY: 0 on: schedule: diff --git a/.github/workflows/call-algolia-deployment-script.yml b/.github/workflows/call-algolia-deployment-script.yml index f2e8794e0b..c589de3022 100644 --- a/.github/workflows/call-algolia-deployment-script.yml +++ b/.github/workflows/call-algolia-deployment-script.yml @@ -1,5 +1,7 @@ # .github/workflows/call-algolia-deployment-script.yml name: Call algolia deployment script on merge +env: + HUSKY: 0 on: push: diff --git a/.github/workflows/check-a11y-of-changed-content.yaml b/.github/workflows/check-a11y-of-changed-content.yaml index 3c3d887adc..ba48e3d43f 100644 --- a/.github/workflows/check-a11y-of-changed-content.yaml +++ b/.github/workflows/check-a11y-of-changed-content.yaml @@ -1,4 +1,6 @@ name: Check accessibility of changed content +env: + HUSKY: 0 on: pull_request: diff --git a/.github/workflows/generate-related.yaml b/.github/workflows/generate-related.yaml index 6975945ac9..4cb3fcbafc 100644 --- a/.github/workflows/generate-related.yaml +++ b/.github/workflows/generate-related.yaml @@ -1,5 +1,7 @@ name: Generate Read More related - +env: + HUSKY: 0 + on: workflow_dispatch: branches: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 4d5bc3c8e3..e098f2086c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,4 +1,6 @@ name: Lint Posts +env: + HUSKY: 0 on: pull_request: diff --git a/.github/workflows/remove-unused-images.yaml b/.github/workflows/remove-unused-images.yaml index 5054b7ce6e..6338cc58f6 100644 --- a/.github/workflows/remove-unused-images.yaml +++ b/.github/workflows/remove-unused-images.yaml @@ -1,4 +1,6 @@ name: Remove Unused Images +env: + HUSKY: 0 on: workflow_dispatch: diff --git a/.husky/pre-push b/.husky/pre-push new file mode 100644 index 0000000000..00c9954f7a --- /dev/null +++ b/.husky/pre-push @@ -0,0 +1 @@ +npm run lint-on-push \ No newline at end of file diff --git a/README.md b/README.md index d33bc152ed..641fd38c31 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,10 @@ Then you can navigate to [localhost][localhost] in your browser. Note that if you performed a _sparse checkout_ as recommended, and if this is your first post, then you won't see any blog posts when the site loads unless you've already added a file for your new blog post. +### Pushing to GitHub + +We now have a pre-push hook defined that will perform linting against changed blog posts before any push command goes ahead. This should ensure that errors in the metadata for posts are caught before they are built, as it can be much harder to determine why your post is not appearing from the pages-build-deployment GitHub action logs. If you have run `npm install` then it should automatically take care of setting up the hooks using [Husky](https://typicode.github.io/husky/). If for any reason this is blocking you from pushing and you really need to, you can skip the hook by running `git push --no-verify`. + ## CI/CD We use GitHub Actions for CI/CD. The workflow definitions are in YAML files diff --git a/lintHelper.js b/lintHelper.js new file mode 100644 index 0000000000..246a460ad3 --- /dev/null +++ b/lintHelper.js @@ -0,0 +1,100 @@ +const matter = require("gray-matter"); +const yaml = require("js-yaml"); +const fs = require("fs"); +const clc = require("cli-color"); + +const MAX_CATEGORIES = 3; + +const errorColour = clc.red.bold; +const warningColour = clc.yellow; + +const logError = (...params) => + console.error(errorColour(...params)); + +const logWarning = (...params) => + console.warn(warningColour(...params)); + +const flatMap = (arr, mapFunc) => + arr.reduce((prev, x) => prev.concat(mapFunc(x)), []); + +const getValidCategories = () => { + const categoriesYaml = yaml.safeLoad( + fs.readFileSync("_data/categories.yml", "utf8") + ); + + const categories = flatMap( + // remove 'Latest Articles' which is a pseudo-category + categoriesYaml.filter(c => c.url.startsWith("/category/")), + // merge category title into sub-categories + c => [c.title].concat(c.subcategories ? c.subcategories : []) + ).map(c => c.toLowerCase()); + + console.log("Valid categories are: " + categories.join(', ')); + + return categories; +}; + +const lintPost = (path, categories) => { + try { + const blogPost = fs.readFileSync(path, "utf8"); + const frontMatter = matter(blogPost); + const frontMatterCats = frontMatter.data.categories; + + let category; + let postCategories; + // if the frontmatter defines a 'category' field: + if (frontMatter.data.category) { + category = frontMatter.data.category.toLowerCase(); + postCategories = [category]; + // if the frontmatter defines a 'categories' field with at least one but no more than 3 values: + + } else if (frontMatterCats && frontMatterCats.length && frontMatterCats.length <= MAX_CATEGORIES) { + postCategories = frontMatter.data.categories.map(c => c.toLowerCase()); + category = postCategories[0]; + } else { + logError("The post " + path + " does not have at least one and no more than " + MAX_CATEGORIES + " categories defined."); + return false; + } + + if (!categories.includes(category)) { + logError( + "The post " + path + " does not have a recognised category" + ); + return false; + } else { + postCategories + .filter(c => !categories.includes(c)) + .forEach(c => logWarning( + "The post " + path + " has an unrecognised category: '" + c + "'. Check spelling or remove/move to tags." + )); + } + + const summary = frontMatter.data.summary; + const pathArray = path.split("/"); + const postDateString = pathArray[pathArray.length - 1].substring(0, 10); + const postDate = new Date(postDateString); + if (postDate > new Date("2018-03-26")) { + // Note _prose.yml specifies 130 characters are needed, so if you change this please also change the instructions + if(!summary) { + logError("The post " + path + " does not have a summary.") + return false; + } + else if (summary.length < 130) { + logWarning( + "The post " + path + " summary length is " + summary.length + ". Recommended minimum length for the summary is 130 characters." + ); + } + } + } catch (e) { + logError(path, e); + return false; + } + return true; + } + + module.exports = { + logError, + logWarning, + getValidCategories, + lintPost + } \ No newline at end of file diff --git a/lintOnPush.js b/lintOnPush.js new file mode 100644 index 0000000000..d6ad1398c7 --- /dev/null +++ b/lintOnPush.js @@ -0,0 +1,76 @@ +const { execSync } = require("child_process"); +const { logError, lintPost, getValidCategories } = require("./lintHelper"); +const readline = require("readline") + +const LINTER_MATCH_PATTERN = /^_posts.*\.(md|markdown|html)$/; +const EMPTY_OBJECT_PATTERN = /^0+$/; + +const getChangedFiles = async () => { + const rl = readline.createInterface({input: process.stdin, crlfDelay: Infinity}); + + const ranges = []; + + for await (const line of rl) { + const [_localRef, localSha, _remoteRef, remoteSha] = line.trim().split(/\s+/); + + if (!localSha || localSha.match(EMPTY_OBJECT_PATTERN)) { + continue; // branch deleted, ignore + } + + if (!remoteSha || remoteSha.match(EMPTY_OBJECT_PATTERN)) { + // new branch + ranges.push(localSha); + } else { + ranges.push(`${remoteSha}..${localSha}`); + } + } + if (ranges.length === 0) { + return []; + } + + let files = []; + for (const range of ranges) { + try { + const diffOutput = execSync( + `git diff --name-only ${range}`, + { encoding: "utf8" } + ); + files.push(...diffOutput.split("\n").filter(file => file.match(LINTER_MATCH_PATTERN))); + } catch { + // ignore empty diffs + } + } + + // Ensure unique list + return [...new Set(files)]; +} + +const lintOnPush = async () => { + const changedFiles = await getChangedFiles(); + + if (changedFiles.length === 0) { + console.log("No relevant post files changed."); + process.exit(0); + } + + console.log("Linting posts to be pushed:", changedFiles); + + let fail = false; + const categories = getValidCategories(); + for (const file of changedFiles) { + if (!lintPost(file, categories)) + { + fail = true; + } + } + + if (fail) { + logError("Push blocked due to linting errors."); + process.exit(1); + } +} + +lintOnPush().catch((err) => { + console.error("Unexpected error in pre-push hook:", err); + process.exit(1); +}); \ No newline at end of file diff --git a/lintPosts.js b/lintPosts.js index f7537e55a1..b1b560caf0 100644 --- a/lintPosts.js +++ b/lintPosts.js @@ -1,22 +1,9 @@ const globby = require("globby"); -const matter = require("gray-matter"); const yaml = require("js-yaml"); const fs = require("fs"); -const clc = require("cli-color"); -const LINTER_MATCH_PATTERN="_posts/**/*.{md,markdown,html}"; -const MAX_CATEGORIES = 3; - -const errorColour = clc.red.bold; -const warningColour = clc.yellow; - -const logError = (...params) => - console.error(errorColour(...params)); +const { logError, getValidCategories, lintPost } = require("./lintHelper"); -const logWarning = (...params) => - console.warn(warningColour(...params)); - -const flatMap = (arr, mapFunc) => - arr.reduce((prev, x) => prev.concat(mapFunc(x)), []); +const LINTER_MATCH_PATTERN="_posts/**/*.{md,markdown,html}"; const lintAuthorsYml = () => { const authorsPath = "_data/authors.yml"; @@ -55,78 +42,15 @@ const lintAuthorsYml = () => { }; const lintPosts = () => { - const categoriesYaml = yaml.safeLoad( - fs.readFileSync("_data/categories.yml", "utf8") - ); - - const categories = flatMap( - // remove 'Latest Articles' which is a pseudo-category - categoriesYaml.filter(c => c.url.startsWith("/category/")), - // merge category title into sub-categories - c => [c.title].concat(c.subcategories ? c.subcategories : []) - ).map(c => c.toLowerCase()); - - console.log("Valid categories are: " + categories.join(', ')); + const categories = getValidCategories(); let fail = false; // lint each blog post globby([LINTER_MATCH_PATTERN]).then(paths => { paths.forEach(path => { - try { - const blogPost = fs.readFileSync(path, "utf8"); - const frontMatter = matter(blogPost); - const frontMatterCats = frontMatter.data.categories; - - let category; - let postCategories; - // if the frontmatter defines a 'category' field: - if (frontMatter.data.category) { - category = frontMatter.data.category.toLowerCase(); - postCategories = [category]; - // if the frontmatter defines a 'categories' field with at least one but no more than 3 values: - - } else if (frontMatterCats && frontMatterCats.length && frontMatterCats.length <= MAX_CATEGORIES) { - postCategories = frontMatter.data.categories.map(c => c.toLowerCase()); - category = postCategories[0]; - } else { - logError("The post " + path + " does not have at least one and no more than " + MAX_CATEGORIES + " categories defined."); - fail = true; - return; - } - - if (!categories.includes(category)) { - logError( - "The post " + path + " does not have a recognised category" - ); - fail = true; - } else { - postCategories - .filter(c => !categories.includes(c)) - .forEach(c => logWarning( - "The post " + path + " has an unrecognised category: '" + c + "'. Check spelling or remove/move to tags." - )); - } - - - const summary = frontMatter.data.summary; - const pathArray = path.split("/"); - const postDateString = pathArray[pathArray.length - 1].substring(0, 10); - const postDate = new Date(postDateString); - if (postDate > new Date("2018-03-26")) { - // Note _prose.yml specifies 130 characters are needed, so if you change this please also change the instructions - if(!summary) { - logError("The post " + path + " does not have a summary.") - fail = true; - } - else if (summary.length < 130) { - logWarning( - "The post " + path + " summary length is " + summary.length + ". Recommended minimum length for the summary is 130 characters." - ); - } - } - } catch (e) { - logError(path, e); + if (!lintPost(path, categories)) + { fail = true; } }); diff --git a/package-lock.json b/package-lock.json index 5ac62e8156..55c171757b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "glob-promise": "^4.2.2", "globby": "^7.1.1", "gray-matter": "^3.1.1", + "husky": "^9.1.7", "js-yaml": "^3.10.0", "markdown-spellcheck": "^1.3.1", "markdown-to-txt": "^2.0.0", @@ -1461,6 +1462,21 @@ "hunspell-tojson": "bin/hunspell-tojson.js" } }, + "node_modules/husky": { + "version": "9.1.7", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", + "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", + "dev": true, + "bin": { + "husky": "bin.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", diff --git a/package.json b/package.json index 4c66482e12..976eb971fe 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "glob-promise": "^4.2.2", "globby": "^7.1.1", "gray-matter": "^3.1.1", + "husky": "^9.1.7", "js-yaml": "^3.10.0", "markdown-spellcheck": "^1.3.1", "markdown-to-txt": "^2.0.0", @@ -32,12 +33,14 @@ }, "scripts": { "lint": "node lintPosts.js", + "lint-on-push": "node lintOnPush.js", "compute-embeddings": "node scripts/generate-related/compute-embeddings.js", "generate-related": "node scripts/generate-related/blog-metadata.js", "remove-unused-images": "node scripts/images/remove-images.js", "spellcheck": "mdspell \"**/ceberhardt/_posts/*.md\" --en-gb -a -n -x -t", "style": "sass --no-source-map --style=compressed scss/style.scss style.css", - "scripts": "uglifyjs scripts/initialise-menu.js scripts/jquery-1.9.1.js scripts/jquery.jscroll-2.2.4.js scripts/load-clap-count.js scripts/elapsed.js scripts/graft-studio/header-scroll.js scripts/graft-studio/jquery.mmenu.all.js scripts/graft-studio/jquery.matchHeight.js node_modules/applause-button/dist/applause-button.js node_modules/cookieconsent/build/cookieconsent.min.js -o script.js" + "scripts": "uglifyjs scripts/initialise-menu.js scripts/jquery-1.9.1.js scripts/jquery.jscroll-2.2.4.js scripts/load-clap-count.js scripts/elapsed.js scripts/graft-studio/header-scroll.js scripts/graft-studio/jquery.mmenu.all.js scripts/graft-studio/jquery.matchHeight.js node_modules/applause-button/dist/applause-button.js node_modules/cookieconsent/build/cookieconsent.min.js -o script.js", + "prepare": "husky || true" }, "homepage": "http://blog.scottlogic.com", "private": true