Rate limit Markdown link checks (#3112)

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 name: CI
 
 on:
-  push: {branches: [master, 'feature.*']}
+  push: {branches: [main, 'feature.*']}
   pull_request:
 
 jobs:

CONTRIBUTING.md

@@ -131,7 +131,7 @@ accepted.
   See [Plain CSS `min()` and `max()`][min-max background] for a good example of
   a Background section.
 
-  [min-max background]: https://github.com/sass/sass/blob/master/accepted/min-max.md#background
+  [min-max background]: https://github.com/sass/sass/blob/main/accepted/min-max.md#background
 
 * **Summary**
 

README.md

@@ -1,4 +1,4 @@
-<h1><img width="200px" alt="Sass" src="https://rawgit.com/sass/sass-site/master/source/assets/img/logos/logo.svg" /></h1>
+<h1><img width="200px" alt="Sass" src="https://rawgit.com/sass/sass-site/main/source/assets/img/logos/logo.svg" /></h1>
 
 [![@SassCSS on Twitter](https://img.shields.io/twitter/follow/SassCSS?label=%40SassCSS&style=social)](https://twitter.com/SassCSS)
 &nbsp;&nbsp;
@@ -96,9 +96,9 @@ This repository isn't an implementation of Sass. Those live in
 * [`accepted/`][], which contains proposals that have been accepted and are
   either implemented or in the process of being implemented.
 
-[`spec/`]: https://github.com/sass/sass/tree/master/spec
-[`proposal/`]: https://github.com/sass/sass/tree/master/proposal
-[`accepted/`]: https://github.com/sass/sass/tree/master/accepted
+[`spec/`]: https://github.com/sass/sass/tree/main/spec
+[`proposal/`]: https://github.com/sass/sass/tree/main/proposal
+[`accepted/`]: https://github.com/sass/sass/tree/main/accepted
 
 Note that this doesn't contain a full specification of Sass. Instead, feature
 specifications are written as needed when a new feature is being designed or

package.json

@@ -1,13 +1,20 @@
 {
   "name": "sass",
   "//": [
-    "This is used to track dependencies for the Travis CI tests run on this ",
-    "repository. For the official Sass npm package, see ",
+    "This is used to track dependencies for the Github Actions tests run on ",
+    "this repository. For the official Sass npm package, see ",
     "https://npmjs.org/package/sass"
   ],
   "engines": {
     "node": ">=14.0.0 <17.0.0"
   },
+  "type": "module",
+  "scripts": {
+    "link-check": "node test/link-check.js",
+    "toc-check": "node test/toc-check.js",
+    "fix": "gts fix",
+    "test": "gts lint && tsc --noEmit && npm run toc-check && npm run link-check"
+  },
   "dependencies": {
     "colors": "^1.3.3",
     "diff": "^4.0.1",
@@ -20,13 +27,6 @@
   "devDependencies": {
     "@types/node": "^14.11.2",
     "gts": "^3.1.0",
-    "standard": "^14.3.4",
     "typescript": "^4.0.3"
-  },
-  "scripts": {
-    "link-check": "node test/link-check.mjs",
-    "toc-check": "node test/toc-check.mjs",
-    "fix": "gts fix && standard --fix",
-    "test": "gts lint && standard && tsc --noEmit && npm run toc-check && npm run link-check"
   }
 }

test/link-check.js

@@ -0,0 +1,99 @@
+import colors from 'colors/safe.js';
+import * as fs from 'fs';
+import glob from 'glob';
+import markdownLinkCheck from 'markdown-link-check';
+import markdownToc from 'markdown-toc';
+import * as path from 'path';
+import {fileURLToPath, pathToFileURL, URL} from 'url';
+
+const files = glob.sync('**/*.md', {
+  ignore: ['node_modules/**/*.md'],
+});
+
+const tocCache = new Map();
+
+function getToc(file) {
+  file = path.normalize(file);
+  let toc = tocCache.get(file);
+  if (toc === undefined) {
+    toc = markdownToc(fs.readFileSync(file).toString(), {}).content;
+    tocCache.set(file, toc);
+  }
+  return toc;
+}
+
+function verifyLinkCheckResults(file, results) {
+  const toc = getToc(file);
+
+  for (const result of results) {
+    const url = new URL(result.link, pathToFileURL(file));
+
+    // A link to another file.
+    if (url.protocol === 'file:' && !result.link.match(/ \(.*\)$/)) {
+      const target = fileURLToPath(url);
+      if (!fs.existsSync(target)) throw Error(`Missing file: ${result.link}`);
+      if (url.hash === '') return;
+      if (getToc(target).includes(url.hash)) return;
+      throw Error(`Dead: ${result.link}`);
+    }
+
+    // A link to a section within this file.
+    if (result.link.match(/^#/)) {
+      if (toc.includes(result.link)) return;
+      throw Error(`Dead: ${result.link}`);
+    }
+
+    // A link to an external website.
+    switch (result.status) {
+      case 'dead':
+        if (result.statusCode === 500) {
+          console.log(colors.yellow(`Server error on target: ${result.link}`));
+          return;
+        }
+        throw Error(`Dead: ${result.link}`);
+      case 'error':
+        throw Error(`Error: ${result.link}`);
+    }
+  }
+}
+
+function runLinkCheck(file, {rateLimit}) {
+  return new Promise((resolve, reject) => {
+    setTimeout(check, rateLimit);
+
+    function check() {
+      markdownLinkCheck(
+        fs.readFileSync(file).toString(),
+        {
+          baseUrl: '',
+          // If Github rate limit is reached, wait 60s and try again.
+          retryOn429: true,
+        },
+        (error, results) => {
+          if (error) {
+            reject(error);
+            return;
+          }
+          try {
+            verifyLinkCheckResults(file, results);
+            resolve();
+          } catch (error) {
+            reject(error);
+          }
+        }
+      );
+    }
+  });
+}
+
+(async () => {
+  for (const file of files) {
+    console.log('Checking links in ' + file);
+    try {
+      await runLinkCheck(file, {rateLimit: 500});
+    } catch (error) {
+      console.log(colors.red(error.message));
+      process.exitCode = 1;
+    }
+  }
+})();

test/link-check.mjs

@@ -1,83 +0,0 @@
-import colors from 'colors/safe.js'
-import * as fs from 'fs'
-import glob from 'glob'
-import toc from 'markdown-toc'
-import * as urlModule from 'url'
-import markdownLinkCheck from 'markdown-link-check'
-import * as path from 'path'
-
-var files = glob.sync('**/*.md', { ignore: ['node_modules/**/*.md'] })
-
-var tocCache = new Map()
-
-function getToc (file) {
-  file = path.normalize(file)
-  if (tocCache.has(file)) {
-    return tocCache.get(file)
-  } else {
-    var result = toc(fs.readFileSync(file).toString()).content
-    tocCache.set(file, result)
-    return result
-  }
-}
-
-files.forEach(function (file) {
-  var markdown = fs.readFileSync(file).toString()
-
-  var dirname = path.dirname(urlModule.fileURLToPath(import.meta.url))
-  markdownLinkCheck(markdown, {
-    retryOn429: true, // Retry if the github rate limit is reached
-    baseUrl: path.basename(dirname) + '/'
-  }, function (err, results) {
-    if (err) {
-      console.error('Error', err)
-      return
-    }
-
-    console.log('Reading: ' + file)
-
-    // Get a list of all headers so we can verify intra-document links.
-    var markdownToc = getToc(file)
-
-    results.forEach(function (result) {
-      var url = new URL(result.link, urlModule.pathToFileURL(file))
-      if (url.protocol === 'file:' && !result.link.match(/ \(.*\)$/)) {
-        var target = urlModule.fileURLToPath(url)
-        if (!fs.existsSync(target)) {
-          process.exitCode = 1
-          console.log(colors.red(`Missing file: ${result.link}`))
-          return
-        }
-
-        if (url.hash === '') return
-        var toc = getToc(target)
-
-        if (toc.includes(url.hash)) return
-        process.exitCode = 1
-        console.log(colors.red(`Dead: ${result.link}`))
-        return
-      }
-
-      if (result.link.match(/^#/)) {
-        if (markdownToc.includes(result.link)) {
-          result.status = 'alive'
-        } else {
-          result.status = 'dead'
-          result.statusCode = 0
-        }
-      }
-
-      if (result.status === 'dead') {
-        if (result.statusCode === 500) {
-          console.log(colors.yellow(`Server error on target: ${result.link}`))
-        } else {
-          process.exitCode = 1
-          console.log(colors.red(`Dead: ${result.link}`))
-        }
-      } else if (result.status === 'error') {
-        process.exitCode = 1
-        console.log(colors.red(`Error: ${result.link}`))
-      }
-    })
-  })
-})

test/toc-check.js

@@ -0,0 +1,32 @@
+import colors from 'colors/safe.js';
+import * as diff from 'diff';
+import * as fs from 'fs';
+
+import * as toc from '../tool/toc.js';
+
+toc.files.forEach(file => {
+  const markdown = fs.readFileSync(file).toString();
+  console.log('Reading: ' + file);
+
+  const currentToc = toc.getCurrent(markdown);
+  if (currentToc === null) {
+    console.log(colors.yellow("File doesn't have a table of contents"));
+    return;
+  }
+
+  const generatedToc = toc.generate(markdown);
+  if (currentToc === generatedToc) return;
+
+  const tocDiff = diff.diffLines(currentToc, generatedToc);
+  tocDiff.forEach(part => {
+    const color = part.added
+      ? colors.green
+      : part.removed
+      ? colors.red
+      : colors.grey;
+    process.stderr.write(color(part.value));
+  });
+  process.stderr.write('\n');
+
+  process.exitCode = 1;
+});

test/toc-check.mjs

@@ -1,30 +0,0 @@
-import colors from 'colors/safe.js'
-import diff from 'diff'
-import * as fs from 'fs'
-
-import * as toc from '../tool/src/toc.mjs'
-
-toc.files.forEach(function (file) {
-  var markdown = fs.readFileSync(file).toString()
-  console.log('Reading: ' + file)
-
-  var currentToc = toc.getCurrent(markdown)
-  if (currentToc == null) {
-    console.log(colors.yellow("File doesn't have a table of contents"))
-    return
-  }
-
-  var generatedToc = toc.generate(markdown)
-  if (currentToc === generatedToc) return
-
-  var tocDiff = diff.diffLines(currentToc, generatedToc)
-  tocDiff.forEach(function (part) {
-    var color = part.added
-      ? colors.green
-      : (part.removed ? colors.red : colors.grey)
-    process.stderr.write(color(part.value))
-  })
-  process.stderr.write('\n')
-
-  process.exitCode = 1
-})

tool/src/toc.mjs

@@ -1,36 +0,0 @@
-import glob from 'glob'
-import markdownToc from 'markdown-toc'
-
-/// Files that may contain tables of contents.
-export const files = glob.sync('**/*.md', {
-  ignore: ['node_modules/**/*.md', '**/*.changes.md']
-})
-
-// Returns the index of the first `*` in `markdown`'s table of contents, or
-// `null` if `markdown` doesn't include a table of contents.
-function findStartIndex (markdown) {
-  var match = markdown.match(/\n## Table of Contents\n/)
-  if (match == null) return null
-  return match.index + match[0].length - 1
-}
-
-// Returns the table of contents in `markdown` if it contains one, or `null`
-// otherwise.
-export function getCurrent (markdown) {
-  var startIndex = findStartIndex(markdown)
-  if (startIndex == null) return null
-
-  var endIndex = markdown.indexOf('## ', startIndex) - 1
-  if (endIndex < 0) return null
-
-  return markdown.substring(startIndex, endIndex).trim()
-}
-
-/// Returns the expected table of contents for `markdown`.
-export function generate (markdown) {
-  return markdownToc(markdown, {
-    filter: (string, _, __) => string.indexOf('Table of Contents') === -1,
-    firsth1: false,
-    bullets: '*'
-  }).content
-}

tool/toc.js

@@ -0,0 +1,38 @@
+import glob from 'glob';
+import markdownToc from 'markdown-toc';
+
+/** Files that may contain tables of contents. */
+export const files = glob.sync('**/*.md', {
+  ignore: ['node_modules/**/*.md', '**/*.changes.md'],
+});
+
+// Returns the index of the first `*` in `markdown`'s table of contents, or
+// `null` if `markdown` doesn't include a table of contents.
+function findStartIndex(markdown) {
+  const match = markdown.match(/\n## Table of Contents\n/);
+  if (match === null || match.index === undefined) return null;
+  return match.index + match[0].length - 1;
+}
+
+/**
+ * Returns the table of contents in `markdown` if it contains one, or `null`
+ * otherwise.
+ */
+export function getCurrent(markdown) {
+  const startIndex = findStartIndex(markdown);
+  if (startIndex === null) return null;
+
+  const endIndex = markdown.indexOf('## ', startIndex) - 1;
+  if (endIndex < 0) return null;
+
+  return markdown.substring(startIndex, endIndex).trim();
+}
+
+/** Returns the expected table of contents for `markdown`. */
+export function generate(markdown) {
+  return markdownToc(markdown, {
+    filter: contents => contents.indexOf('Table of Contents') === -1,
+    firsth1: false,
+    bullets: '*',
+  }).content;
+}

tool/update-toc.js

@@ -0,0 +1,32 @@
+import * as fs from 'fs';
+
+import * as toc from './toc.js';
+
+toc.files.forEach(file => {
+  const markdown = fs.readFileSync(file).toString();
+
+  const currentToc = toc.getCurrent(markdown);
+  if (currentToc === null) {
+    const match = markdown.match('## Table of Contents\n\n');
+    if (!match || !match.index) return;
+
+    const tocLocation = match.index + match[0].length;
+
+    // If there's an empty TOC, fill it in.
+    fs.writeFileSync(
+      file,
+      markdown.substring(0, tocLocation) +
+        toc.generate(markdown) +
+        '\n\n' +
+        markdown.substring(tocLocation)
+    );
+    console.log(`Added TOC to ${file}`);
+    return;
+  }
+
+  const generatedToc = toc.generate(markdown);
+  if (currentToc === generatedToc) return;
+
+  fs.writeFileSync(file, markdown.replace(currentToc, generatedToc));
+  console.log(`Updated TOC in ${file}`);
+});

tool/update-toc.mjs

@@ -1,29 +0,0 @@
-import * as toc from './src/toc.mjs'
-import * as fs from 'fs'
-
-toc.files.forEach(function (file) {
-  var markdown = fs.readFileSync(file).toString()
-
-  var currentToc = toc.getCurrent(markdown)
-  if (currentToc == null) {
-    var match = markdown.match('## Table of Contents\n\n')
-    if (!match) return
-
-    var tocLocation = match.index + match[0].length
-
-    // If there's an empty TOC, fill it in.
-    fs.writeFileSync(
-      file,
-      markdown.substring(0, tocLocation) +
-        toc.generate(markdown) + '\n\n' +
-        markdown.substring(tocLocation))
-    console.log(`Added TOC to ${file}`)
-    return
-  }
-
-  var generatedToc = toc.generate(markdown)
-  if (currentToc === generatedToc) return
-
-  fs.writeFileSync(file, markdown.replace(currentToc, generatedToc))
-  console.log(`Updated TOC in ${file}`)
-})

tsconfig.json

@@ -1,10 +1,8 @@
 {
   "extends": "./node_modules/gts/tsconfig-google.json",
-  "compilerOptions": {"rootDir": "."},
-  "include": [
-    "index.d.ts",
-    "spec/**/*.ts",
-    "proposal/**/*.ts",
-    "accepted/**/*.ts"
-  ]
+  "compilerOptions": {
+    "allowJs": true,
+    "rootDir": "."
+  },
+  "include": ["accepted/**/*.ts", "proposal/**/*.ts", "spec/**/*.ts"]
 }