From 1127741bd1fc91e8f037157676db11bd09597361 Mon Sep 17 00:00:00 2001 From: Matthew Griffin Date: Wed, 23 Jul 2025 14:59:32 +0100 Subject: [PATCH 1/6] Added husky to support linting as posts are committed --- .husky/pre-commit | 1 + lintCommit.js | 37 +++++++++++++++++ lintHelper.js | 100 ++++++++++++++++++++++++++++++++++++++++++++++ lintPosts.js | 86 +++------------------------------------ package-lock.json | 16 ++++++++ package.json | 5 ++- 6 files changed, 163 insertions(+), 82 deletions(-) create mode 100644 .husky/pre-commit create mode 100644 lintCommit.js create mode 100644 lintHelper.js diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100644 index 0000000000..5cd574fae7 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1 @@ +npm run lint-commit \ No newline at end of file diff --git a/lintCommit.js b/lintCommit.js new file mode 100644 index 0000000000..d9d84f05b4 --- /dev/null +++ b/lintCommit.js @@ -0,0 +1,37 @@ +const { execSync } = require("child_process"); +const { logError, lintPost, getValidCategories } = require("./lintHelper"); + +const LINTER_MATCH_PATTERN = /^_posts.*\.(md|markdown|html)$/; + +const lintCommit = () => { + const categories = getValidCategories(); + + const changedFiles = execSync("git diff --cached --name-only", { + encoding: "utf-8", + }) + .split("\n") + .filter((file) => file.match(LINTER_MATCH_PATTERN)); + + if (changedFiles.length === 0) { + console.log("No relevant post files changed."); + process.exit(0); + } + + console.log("Linting posts to be committed:", changedFiles); + + let fail = false; + + for (const file of changedFiles) { + if (!lintPost(file, categories)) + { + fail = true; + } + } + + if (fail) { + logError("Commit blocked due to linting errors."); + process.exit(1); + } +} + +lintCommit() \ No newline at end of file diff --git a/lintHelper.js b/lintHelper.js new file mode 100644 index 0000000000..246a460ad3 --- /dev/null +++ b/lintHelper.js @@ -0,0 +1,100 @@ +const matter = require("gray-matter"); +const yaml = require("js-yaml"); +const fs = require("fs"); +const clc = require("cli-color"); + +const MAX_CATEGORIES = 3; + +const errorColour = clc.red.bold; +const warningColour = clc.yellow; + +const logError = (...params) => + console.error(errorColour(...params)); + +const logWarning = (...params) => + console.warn(warningColour(...params)); + +const flatMap = (arr, mapFunc) => + arr.reduce((prev, x) => prev.concat(mapFunc(x)), []); + +const getValidCategories = () => { + const categoriesYaml = yaml.safeLoad( + fs.readFileSync("_data/categories.yml", "utf8") + ); + + const categories = flatMap( + // remove 'Latest Articles' which is a pseudo-category + categoriesYaml.filter(c => c.url.startsWith("/category/")), + // merge category title into sub-categories + c => [c.title].concat(c.subcategories ? c.subcategories : []) + ).map(c => c.toLowerCase()); + + console.log("Valid categories are: " + categories.join(', ')); + + return categories; +}; + +const lintPost = (path, categories) => { + try { + const blogPost = fs.readFileSync(path, "utf8"); + const frontMatter = matter(blogPost); + const frontMatterCats = frontMatter.data.categories; + + let category; + let postCategories; + // if the frontmatter defines a 'category' field: + if (frontMatter.data.category) { + category = frontMatter.data.category.toLowerCase(); + postCategories = [category]; + // if the frontmatter defines a 'categories' field with at least one but no more than 3 values: + + } else if (frontMatterCats && frontMatterCats.length && frontMatterCats.length <= MAX_CATEGORIES) { + postCategories = frontMatter.data.categories.map(c => c.toLowerCase()); + category = postCategories[0]; + } else { + logError("The post " + path + " does not have at least one and no more than " + MAX_CATEGORIES + " categories defined."); + return false; + } + + if (!categories.includes(category)) { + logError( + "The post " + path + " does not have a recognised category" + ); + return false; + } else { + postCategories + .filter(c => !categories.includes(c)) + .forEach(c => logWarning( + "The post " + path + " has an unrecognised category: '" + c + "'. Check spelling or remove/move to tags." + )); + } + + const summary = frontMatter.data.summary; + const pathArray = path.split("/"); + const postDateString = pathArray[pathArray.length - 1].substring(0, 10); + const postDate = new Date(postDateString); + if (postDate > new Date("2018-03-26")) { + // Note _prose.yml specifies 130 characters are needed, so if you change this please also change the instructions + if(!summary) { + logError("The post " + path + " does not have a summary.") + return false; + } + else if (summary.length < 130) { + logWarning( + "The post " + path + " summary length is " + summary.length + ". Recommended minimum length for the summary is 130 characters." + ); + } + } + } catch (e) { + logError(path, e); + return false; + } + return true; + } + + module.exports = { + logError, + logWarning, + getValidCategories, + lintPost + } \ No newline at end of file diff --git a/lintPosts.js b/lintPosts.js index f7537e55a1..b1b560caf0 100644 --- a/lintPosts.js +++ b/lintPosts.js @@ -1,22 +1,9 @@ const globby = require("globby"); -const matter = require("gray-matter"); const yaml = require("js-yaml"); const fs = require("fs"); -const clc = require("cli-color"); -const LINTER_MATCH_PATTERN="_posts/**/*.{md,markdown,html}"; -const MAX_CATEGORIES = 3; - -const errorColour = clc.red.bold; -const warningColour = clc.yellow; - -const logError = (...params) => - console.error(errorColour(...params)); +const { logError, getValidCategories, lintPost } = require("./lintHelper"); -const logWarning = (...params) => - console.warn(warningColour(...params)); - -const flatMap = (arr, mapFunc) => - arr.reduce((prev, x) => prev.concat(mapFunc(x)), []); +const LINTER_MATCH_PATTERN="_posts/**/*.{md,markdown,html}"; const lintAuthorsYml = () => { const authorsPath = "_data/authors.yml"; @@ -55,78 +42,15 @@ const lintAuthorsYml = () => { }; const lintPosts = () => { - const categoriesYaml = yaml.safeLoad( - fs.readFileSync("_data/categories.yml", "utf8") - ); - - const categories = flatMap( - // remove 'Latest Articles' which is a pseudo-category - categoriesYaml.filter(c => c.url.startsWith("/category/")), - // merge category title into sub-categories - c => [c.title].concat(c.subcategories ? c.subcategories : []) - ).map(c => c.toLowerCase()); - - console.log("Valid categories are: " + categories.join(', ')); + const categories = getValidCategories(); let fail = false; // lint each blog post globby([LINTER_MATCH_PATTERN]).then(paths => { paths.forEach(path => { - try { - const blogPost = fs.readFileSync(path, "utf8"); - const frontMatter = matter(blogPost); - const frontMatterCats = frontMatter.data.categories; - - let category; - let postCategories; - // if the frontmatter defines a 'category' field: - if (frontMatter.data.category) { - category = frontMatter.data.category.toLowerCase(); - postCategories = [category]; - // if the frontmatter defines a 'categories' field with at least one but no more than 3 values: - - } else if (frontMatterCats && frontMatterCats.length && frontMatterCats.length <= MAX_CATEGORIES) { - postCategories = frontMatter.data.categories.map(c => c.toLowerCase()); - category = postCategories[0]; - } else { - logError("The post " + path + " does not have at least one and no more than " + MAX_CATEGORIES + " categories defined."); - fail = true; - return; - } - - if (!categories.includes(category)) { - logError( - "The post " + path + " does not have a recognised category" - ); - fail = true; - } else { - postCategories - .filter(c => !categories.includes(c)) - .forEach(c => logWarning( - "The post " + path + " has an unrecognised category: '" + c + "'. Check spelling or remove/move to tags." - )); - } - - - const summary = frontMatter.data.summary; - const pathArray = path.split("/"); - const postDateString = pathArray[pathArray.length - 1].substring(0, 10); - const postDate = new Date(postDateString); - if (postDate > new Date("2018-03-26")) { - // Note _prose.yml specifies 130 characters are needed, so if you change this please also change the instructions - if(!summary) { - logError("The post " + path + " does not have a summary.") - fail = true; - } - else if (summary.length < 130) { - logWarning( - "The post " + path + " summary length is " + summary.length + ". Recommended minimum length for the summary is 130 characters." - ); - } - } - } catch (e) { - logError(path, e); + if (!lintPost(path, categories)) + { fail = true; } }); diff --git a/package-lock.json b/package-lock.json index 5ac62e8156..55c171757b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "glob-promise": "^4.2.2", "globby": "^7.1.1", "gray-matter": "^3.1.1", + "husky": "^9.1.7", "js-yaml": "^3.10.0", "markdown-spellcheck": "^1.3.1", "markdown-to-txt": "^2.0.0", @@ -1461,6 +1462,21 @@ "hunspell-tojson": "bin/hunspell-tojson.js" } }, + "node_modules/husky": { + "version": "9.1.7", + "resolved": "https://registry.npmjs.org/husky/-/husky-9.1.7.tgz", + "integrity": "sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==", + "dev": true, + "bin": { + "husky": "bin.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", diff --git a/package.json b/package.json index 4c66482e12..624a9deb23 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "glob-promise": "^4.2.2", "globby": "^7.1.1", "gray-matter": "^3.1.1", + "husky": "^9.1.7", "js-yaml": "^3.10.0", "markdown-spellcheck": "^1.3.1", "markdown-to-txt": "^2.0.0", @@ -32,12 +33,14 @@ }, "scripts": { "lint": "node lintPosts.js", + "lint-commit": "node lintCommit.js", "compute-embeddings": "node scripts/generate-related/compute-embeddings.js", "generate-related": "node scripts/generate-related/blog-metadata.js", "remove-unused-images": "node scripts/images/remove-images.js", "spellcheck": "mdspell \"**/ceberhardt/_posts/*.md\" --en-gb -a -n -x -t", "style": "sass --no-source-map --style=compressed scss/style.scss style.css", - "scripts": "uglifyjs scripts/initialise-menu.js scripts/jquery-1.9.1.js scripts/jquery.jscroll-2.2.4.js scripts/load-clap-count.js scripts/elapsed.js scripts/graft-studio/header-scroll.js scripts/graft-studio/jquery.mmenu.all.js scripts/graft-studio/jquery.matchHeight.js node_modules/applause-button/dist/applause-button.js node_modules/cookieconsent/build/cookieconsent.min.js -o script.js" + "scripts": "uglifyjs scripts/initialise-menu.js scripts/jquery-1.9.1.js scripts/jquery.jscroll-2.2.4.js scripts/load-clap-count.js scripts/elapsed.js scripts/graft-studio/header-scroll.js scripts/graft-studio/jquery.mmenu.all.js scripts/graft-studio/jquery.matchHeight.js node_modules/applause-button/dist/applause-button.js node_modules/cookieconsent/build/cookieconsent.min.js -o script.js", + "prepare": "husky" }, "homepage": "http://blog.scottlogic.com", "private": true From 323172c183a7c89e8a8d5e18d2f69d43852e77e0 Mon Sep 17 00:00:00 2001 From: Matthew Griffin Date: Wed, 23 Jul 2025 15:05:57 +0100 Subject: [PATCH 2/6] Don't initialise husky in CI builds --- .github/workflows/auto-compress-images.yaml | 2 ++ .github/workflows/call-algolia-deployment-script.yml | 2 ++ .github/workflows/check-a11y-of-changed-content.yaml | 2 ++ .github/workflows/generate-related.yaml | 4 +++- .github/workflows/lint.yaml | 2 ++ .github/workflows/remove-unused-images.yaml | 2 ++ package.json | 2 +- 7 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/auto-compress-images.yaml b/.github/workflows/auto-compress-images.yaml index 4c0e74b12f..c827e28056 100644 --- a/.github/workflows/auto-compress-images.yaml +++ b/.github/workflows/auto-compress-images.yaml @@ -1,4 +1,6 @@ name: Compress Images Once a Month - Creates Pull Request +env: + HUSKY: 0 on: schedule: diff --git a/.github/workflows/call-algolia-deployment-script.yml b/.github/workflows/call-algolia-deployment-script.yml index f2e8794e0b..c589de3022 100644 --- a/.github/workflows/call-algolia-deployment-script.yml +++ b/.github/workflows/call-algolia-deployment-script.yml @@ -1,5 +1,7 @@ # .github/workflows/call-algolia-deployment-script.yml name: Call algolia deployment script on merge +env: + HUSKY: 0 on: push: diff --git a/.github/workflows/check-a11y-of-changed-content.yaml b/.github/workflows/check-a11y-of-changed-content.yaml index 3c3d887adc..ba48e3d43f 100644 --- a/.github/workflows/check-a11y-of-changed-content.yaml +++ b/.github/workflows/check-a11y-of-changed-content.yaml @@ -1,4 +1,6 @@ name: Check accessibility of changed content +env: + HUSKY: 0 on: pull_request: diff --git a/.github/workflows/generate-related.yaml b/.github/workflows/generate-related.yaml index 6975945ac9..4cb3fcbafc 100644 --- a/.github/workflows/generate-related.yaml +++ b/.github/workflows/generate-related.yaml @@ -1,5 +1,7 @@ name: Generate Read More related - +env: + HUSKY: 0 + on: workflow_dispatch: branches: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 4d5bc3c8e3..e098f2086c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,4 +1,6 @@ name: Lint Posts +env: + HUSKY: 0 on: pull_request: diff --git a/.github/workflows/remove-unused-images.yaml b/.github/workflows/remove-unused-images.yaml index 5054b7ce6e..6338cc58f6 100644 --- a/.github/workflows/remove-unused-images.yaml +++ b/.github/workflows/remove-unused-images.yaml @@ -1,4 +1,6 @@ name: Remove Unused Images +env: + HUSKY: 0 on: workflow_dispatch: diff --git a/package.json b/package.json index 624a9deb23..c73cddc4ea 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "spellcheck": "mdspell \"**/ceberhardt/_posts/*.md\" --en-gb -a -n -x -t", "style": "sass --no-source-map --style=compressed scss/style.scss style.css", "scripts": "uglifyjs scripts/initialise-menu.js scripts/jquery-1.9.1.js scripts/jquery.jscroll-2.2.4.js scripts/load-clap-count.js scripts/elapsed.js scripts/graft-studio/header-scroll.js scripts/graft-studio/jquery.mmenu.all.js scripts/graft-studio/jquery.matchHeight.js node_modules/applause-button/dist/applause-button.js node_modules/cookieconsent/build/cookieconsent.min.js -o script.js", - "prepare": "husky" + "prepare": "husky || true" }, "homepage": "http://blog.scottlogic.com", "private": true From 64ebf44f42ffa4d890749f0e7815e0a5b28e1b2f Mon Sep 17 00:00:00 2001 From: Matthew Griffin Date: Mon, 15 Sep 2025 14:40:46 +0100 Subject: [PATCH 3/6] Changed linting to a pre-push check --- .husky/pre-commit | 1 - .husky/pre-push | 1 + lintCommit.js | 37 ---------------------- lintOnPush.js | 80 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 5 files changed, 82 insertions(+), 39 deletions(-) delete mode 100644 .husky/pre-commit create mode 100644 .husky/pre-push delete mode 100644 lintCommit.js create mode 100644 lintOnPush.js diff --git a/.husky/pre-commit b/.husky/pre-commit deleted file mode 100644 index 5cd574fae7..0000000000 --- a/.husky/pre-commit +++ /dev/null @@ -1 +0,0 @@ -npm run lint-commit \ No newline at end of file diff --git a/.husky/pre-push b/.husky/pre-push new file mode 100644 index 0000000000..00c9954f7a --- /dev/null +++ b/.husky/pre-push @@ -0,0 +1 @@ +npm run lint-on-push \ No newline at end of file diff --git a/lintCommit.js b/lintCommit.js deleted file mode 100644 index d9d84f05b4..0000000000 --- a/lintCommit.js +++ /dev/null @@ -1,37 +0,0 @@ -const { execSync } = require("child_process"); -const { logError, lintPost, getValidCategories } = require("./lintHelper"); - -const LINTER_MATCH_PATTERN = /^_posts.*\.(md|markdown|html)$/; - -const lintCommit = () => { - const categories = getValidCategories(); - - const changedFiles = execSync("git diff --cached --name-only", { - encoding: "utf-8", - }) - .split("\n") - .filter((file) => file.match(LINTER_MATCH_PATTERN)); - - if (changedFiles.length === 0) { - console.log("No relevant post files changed."); - process.exit(0); - } - - console.log("Linting posts to be committed:", changedFiles); - - let fail = false; - - for (const file of changedFiles) { - if (!lintPost(file, categories)) - { - fail = true; - } - } - - if (fail) { - logError("Commit blocked due to linting errors."); - process.exit(1); - } -} - -lintCommit() \ No newline at end of file diff --git a/lintOnPush.js b/lintOnPush.js new file mode 100644 index 0000000000..b5d9a7827b --- /dev/null +++ b/lintOnPush.js @@ -0,0 +1,80 @@ +const { execSync } = require("child_process"); +const { logError, lintPost, getValidCategories } = require("./lintHelper"); +const readline = require("readline") + +const LINTER_MATCH_PATTERN = /^_posts.*\.(md|markdown|html)$/; +const EMPTY_OBJECT_PATTERN = /^0+$/; + +const getChangedFiles = async () => { + const rl = readline.createInterface({input: process.stdin, crlfDelay: Infinity}); + + const ranges = []; + + for await (const line of rl) { + const [_localRef, localSha, _remoteRef, remoteSha] = line.trim().split(/\s+/); + + if (!localSha || localSha.match(EMPTY_OBJECT_PATTERN)) { + continue; // branch deleted, ignore + } + + if (!remoteSha || remoteSha.match(EMPTY_OBJECT_PATTERN)) { + // new branch + ranges.push(localSha); + } else { + ranges.push(`${remoteSha}..${localSha}`); + } + } + if (ranges.length === 0) { + return []; + } + + let files = []; + for (const range of ranges) { + try { + const diffOutput = execSync( + `git diff --name-only ${range}`, + { encoding: "utf8" } + ); + files.push(...diffOutput.split("\n").filter(file => file.match(LINTER_MATCH_PATTERN))); + } catch { + // ignore empty diffs + } + } + + return new Set(files); +} + +const lintOnPush = async () => { + const categories = getValidCategories(); + + const changedFiles = await getChangedFiles(); + + if (changedFiles.length === 0) { + console.log("No relevant post files changed."); + process.exit(0); + } + + console.log("Linting posts to be committed:", changedFiles); + + let fail = false; + + for (const file of changedFiles) { + if (!lintPost(file, categories)) + { + fail = true; + } + } + + if (fail) { + logError("Commit blocked due to linting errors."); + process.exit(1); + } +} + +lintOnPush().then(() => { + console.log("Linting completed successfully"); + process.exit(1); +}).catch((err) => { + console.error("Unexpected error in pre-push hook:", err); + process.exit(1); +}); \ No newline at end of file diff --git a/package.json b/package.json index c73cddc4ea..976eb971fe 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ }, "scripts": { "lint": "node lintPosts.js", - "lint-commit": "node lintCommit.js", + "lint-on-push": "node lintOnPush.js", "compute-embeddings": "node scripts/generate-related/compute-embeddings.js", "generate-related": "node scripts/generate-related/blog-metadata.js", "remove-unused-images": "node scripts/images/remove-images.js", From 083bc188242df9dd391edf0d6d2241d7210aa4aa Mon Sep 17 00:00:00 2001 From: Matthew Griffin Date: Mon, 15 Sep 2025 14:47:35 +0100 Subject: [PATCH 4/6] Ensure array is always returned Remove testing code --- lintOnPush.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lintOnPush.js b/lintOnPush.js index b5d9a7827b..9b13461a1d 100644 --- a/lintOnPush.js +++ b/lintOnPush.js @@ -41,7 +41,8 @@ const getChangedFiles = async () => { } } - return new Set(files); + // Ensure unique list + return [...new Set(files)]; } const lintOnPush = async () => { @@ -71,10 +72,7 @@ const lintOnPush = async () => { } } -lintOnPush().then(() => { - console.log("Linting completed successfully"); - process.exit(1); -}).catch((err) => { +lintOnPush().catch((err) => { console.error("Unexpected error in pre-push hook:", err); process.exit(1); }); \ No newline at end of file From 40fd9019fdda62842299ddfc7383727cafbf739c Mon Sep 17 00:00:00 2001 From: Matthew Griffin Date: Mon, 15 Sep 2025 15:21:42 +0100 Subject: [PATCH 5/6] Added details about the pre-push hook to the README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d33bc152ed..641fd38c31 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,10 @@ Then you can navigate to [localhost][localhost] in your browser. Note that if you performed a _sparse checkout_ as recommended, and if this is your first post, then you won't see any blog posts when the site loads unless you've already added a file for your new blog post. +### Pushing to GitHub + +We now have a pre-push hook defined that will perform linting against changed blog posts before any push command goes ahead. This should ensure that errors in the metadata for posts are caught before they are built, as it can be much harder to determine why your post is not appearing from the pages-build-deployment GitHub action logs. If you have run `npm install` then it should automatically take care of setting up the hooks using [Husky](https://typicode.github.io/husky/). If for any reason this is blocking you from pushing and you really need to, you can skip the hook by running `git push --no-verify`. + ## CI/CD We use GitHub Actions for CI/CD. The workflow definitions are in YAML files From bf37b0d9ca847d3295b0f53d25af6ffc881a33d0 Mon Sep 17 00:00:00 2001 From: Matthew Griffin Date: Mon, 15 Sep 2025 15:26:44 +0100 Subject: [PATCH 6/6] Fixed log messages that still referred to commit Moved category lookup to after early out when no changes --- lintOnPush.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lintOnPush.js b/lintOnPush.js index 9b13461a1d..d6ad1398c7 100644 --- a/lintOnPush.js +++ b/lintOnPush.js @@ -46,8 +46,6 @@ const getChangedFiles = async () => { } const lintOnPush = async () => { - const categories = getValidCategories(); - const changedFiles = await getChangedFiles(); if (changedFiles.length === 0) { @@ -55,10 +53,10 @@ const lintOnPush = async () => { process.exit(0); } - console.log("Linting posts to be committed:", changedFiles); + console.log("Linting posts to be pushed:", changedFiles); let fail = false; - + const categories = getValidCategories(); for (const file of changedFiles) { if (!lintPost(file, categories)) { @@ -67,7 +65,7 @@ const lintOnPush = async () => { } if (fail) { - logError("Commit blocked due to linting errors."); + logError("Push blocked due to linting errors."); process.exit(1); } }