mirror of
https://github.com/sass/sass.git
synced 2024-09-21 02:27:30 +00:00
Rate limit Markdown link checks (#3112)
This commit is contained in:
parent
0c11d8576f
commit
7cd83369e9
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@ -1,7 +1,7 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push: {branches: [master, 'feature.*']}
|
||||
push: {branches: [main, 'feature.*']}
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
|
@ -131,7 +131,7 @@ accepted.
|
||||
See [Plain CSS `min()` and `max()`][min-max background] for a good example of
|
||||
a Background section.
|
||||
|
||||
[min-max background]: https://github.com/sass/sass/blob/master/accepted/min-max.md#background
|
||||
[min-max background]: https://github.com/sass/sass/blob/main/accepted/min-max.md#background
|
||||
|
||||
* **Summary**
|
||||
|
||||
|
@ -1,4 +1,4 @@
|
||||
<h1><img width="200px" alt="Sass" src="https://rawgit.com/sass/sass-site/master/source/assets/img/logos/logo.svg" /></h1>
|
||||
<h1><img width="200px" alt="Sass" src="https://rawgit.com/sass/sass-site/main/source/assets/img/logos/logo.svg" /></h1>
|
||||
|
||||
[![@SassCSS on Twitter](https://img.shields.io/twitter/follow/SassCSS?label=%40SassCSS&style=social)](https://twitter.com/SassCSS)
|
||||
|
||||
@ -96,9 +96,9 @@ This repository isn't an implementation of Sass. Those live in
|
||||
* [`accepted/`][], which contains proposals that have been accepted and are
|
||||
either implemented or in the process of being implemented.
|
||||
|
||||
[`spec/`]: https://github.com/sass/sass/tree/master/spec
|
||||
[`proposal/`]: https://github.com/sass/sass/tree/master/proposal
|
||||
[`accepted/`]: https://github.com/sass/sass/tree/master/accepted
|
||||
[`spec/`]: https://github.com/sass/sass/tree/main/spec
|
||||
[`proposal/`]: https://github.com/sass/sass/tree/main/proposal
|
||||
[`accepted/`]: https://github.com/sass/sass/tree/main/accepted
|
||||
|
||||
Note that this doesn't contain a full specification of Sass. Instead, feature
|
||||
specifications are written as needed when a new feature is being designed or
|
||||
|
3399
package-lock.json
generated
3399
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
18
package.json
18
package.json
@ -1,13 +1,20 @@
|
||||
{
|
||||
"name": "sass",
|
||||
"//": [
|
||||
"This is used to track dependencies for the Travis CI tests run on this ",
|
||||
"repository. For the official Sass npm package, see ",
|
||||
"This is used to track dependencies for the Github Actions tests run on ",
|
||||
"this repository. For the official Sass npm package, see ",
|
||||
"https://npmjs.org/package/sass"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=14.0.0 <17.0.0"
|
||||
},
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"link-check": "node test/link-check.js",
|
||||
"toc-check": "node test/toc-check.js",
|
||||
"fix": "gts fix",
|
||||
"test": "gts lint && tsc --noEmit && npm run toc-check && npm run link-check"
|
||||
},
|
||||
"dependencies": {
|
||||
"colors": "^1.3.3",
|
||||
"diff": "^4.0.1",
|
||||
@ -20,13 +27,6 @@
|
||||
"devDependencies": {
|
||||
"@types/node": "^14.11.2",
|
||||
"gts": "^3.1.0",
|
||||
"standard": "^14.3.4",
|
||||
"typescript": "^4.0.3"
|
||||
},
|
||||
"scripts": {
|
||||
"link-check": "node test/link-check.mjs",
|
||||
"toc-check": "node test/toc-check.mjs",
|
||||
"fix": "gts fix && standard --fix",
|
||||
"test": "gts lint && standard && tsc --noEmit && npm run toc-check && npm run link-check"
|
||||
}
|
||||
}
|
||||
|
99
test/link-check.js
Normal file
99
test/link-check.js
Normal file
@ -0,0 +1,99 @@
|
||||
import colors from 'colors/safe.js';
|
||||
import * as fs from 'fs';
|
||||
import glob from 'glob';
|
||||
import markdownLinkCheck from 'markdown-link-check';
|
||||
import markdownToc from 'markdown-toc';
|
||||
import * as path from 'path';
|
||||
import {fileURLToPath, pathToFileURL, URL} from 'url';
|
||||
|
||||
const files = glob.sync('**/*.md', {
|
||||
ignore: ['node_modules/**/*.md'],
|
||||
});
|
||||
|
||||
const tocCache = new Map();
|
||||
|
||||
function getToc(file) {
|
||||
file = path.normalize(file);
|
||||
let toc = tocCache.get(file);
|
||||
if (toc === undefined) {
|
||||
toc = markdownToc(fs.readFileSync(file).toString(), {}).content;
|
||||
tocCache.set(file, toc);
|
||||
}
|
||||
return toc;
|
||||
}
|
||||
|
||||
function verifyLinkCheckResults(file, results) {
|
||||
const toc = getToc(file);
|
||||
|
||||
for (const result of results) {
|
||||
const url = new URL(result.link, pathToFileURL(file));
|
||||
|
||||
// A link to another file.
|
||||
if (url.protocol === 'file:' && !result.link.match(/ \(.*\)$/)) {
|
||||
const target = fileURLToPath(url);
|
||||
if (!fs.existsSync(target)) throw Error(`Missing file: ${result.link}`);
|
||||
if (url.hash === '') return;
|
||||
if (getToc(target).includes(url.hash)) return;
|
||||
throw Error(`Dead: ${result.link}`);
|
||||
}
|
||||
|
||||
// A link to a section within this file.
|
||||
if (result.link.match(/^#/)) {
|
||||
if (toc.includes(result.link)) return;
|
||||
throw Error(`Dead: ${result.link}`);
|
||||
}
|
||||
|
||||
// A link to an external website.
|
||||
switch (result.status) {
|
||||
case 'dead':
|
||||
if (result.statusCode === 500) {
|
||||
console.log(colors.yellow(`Server error on target: ${result.link}`));
|
||||
return;
|
||||
}
|
||||
throw Error(`Dead: ${result.link}`);
|
||||
case 'error':
|
||||
throw Error(`Error: ${result.link}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runLinkCheck(file, {rateLimit}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
setTimeout(check, rateLimit);
|
||||
|
||||
function check() {
|
||||
markdownLinkCheck(
|
||||
fs.readFileSync(file).toString(),
|
||||
{
|
||||
baseUrl: '',
|
||||
// If Github rate limit is reached, wait 60s and try again.
|
||||
retryOn429: true,
|
||||
},
|
||||
(error, results) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
verifyLinkCheckResults(file, results);
|
||||
resolve();
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
(async () => {
|
||||
for (const file of files) {
|
||||
console.log('Checking links in ' + file);
|
||||
try {
|
||||
await runLinkCheck(file, {rateLimit: 500});
|
||||
} catch (error) {
|
||||
console.log(colors.red(error.message));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
})();
|
@ -1,83 +0,0 @@
|
||||
import colors from 'colors/safe.js'
|
||||
import * as fs from 'fs'
|
||||
import glob from 'glob'
|
||||
import toc from 'markdown-toc'
|
||||
import * as urlModule from 'url'
|
||||
import markdownLinkCheck from 'markdown-link-check'
|
||||
import * as path from 'path'
|
||||
|
||||
var files = glob.sync('**/*.md', { ignore: ['node_modules/**/*.md'] })
|
||||
|
||||
var tocCache = new Map()
|
||||
|
||||
function getToc (file) {
|
||||
file = path.normalize(file)
|
||||
if (tocCache.has(file)) {
|
||||
return tocCache.get(file)
|
||||
} else {
|
||||
var result = toc(fs.readFileSync(file).toString()).content
|
||||
tocCache.set(file, result)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
files.forEach(function (file) {
|
||||
var markdown = fs.readFileSync(file).toString()
|
||||
|
||||
var dirname = path.dirname(urlModule.fileURLToPath(import.meta.url))
|
||||
markdownLinkCheck(markdown, {
|
||||
retryOn429: true, // Retry if the github rate limit is reached
|
||||
baseUrl: path.basename(dirname) + '/'
|
||||
}, function (err, results) {
|
||||
if (err) {
|
||||
console.error('Error', err)
|
||||
return
|
||||
}
|
||||
|
||||
console.log('Reading: ' + file)
|
||||
|
||||
// Get a list of all headers so we can verify intra-document links.
|
||||
var markdownToc = getToc(file)
|
||||
|
||||
results.forEach(function (result) {
|
||||
var url = new URL(result.link, urlModule.pathToFileURL(file))
|
||||
if (url.protocol === 'file:' && !result.link.match(/ \(.*\)$/)) {
|
||||
var target = urlModule.fileURLToPath(url)
|
||||
if (!fs.existsSync(target)) {
|
||||
process.exitCode = 1
|
||||
console.log(colors.red(`Missing file: ${result.link}`))
|
||||
return
|
||||
}
|
||||
|
||||
if (url.hash === '') return
|
||||
var toc = getToc(target)
|
||||
|
||||
if (toc.includes(url.hash)) return
|
||||
process.exitCode = 1
|
||||
console.log(colors.red(`Dead: ${result.link}`))
|
||||
return
|
||||
}
|
||||
|
||||
if (result.link.match(/^#/)) {
|
||||
if (markdownToc.includes(result.link)) {
|
||||
result.status = 'alive'
|
||||
} else {
|
||||
result.status = 'dead'
|
||||
result.statusCode = 0
|
||||
}
|
||||
}
|
||||
|
||||
if (result.status === 'dead') {
|
||||
if (result.statusCode === 500) {
|
||||
console.log(colors.yellow(`Server error on target: ${result.link}`))
|
||||
} else {
|
||||
process.exitCode = 1
|
||||
console.log(colors.red(`Dead: ${result.link}`))
|
||||
}
|
||||
} else if (result.status === 'error') {
|
||||
process.exitCode = 1
|
||||
console.log(colors.red(`Error: ${result.link}`))
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
32
test/toc-check.js
Normal file
32
test/toc-check.js
Normal file
@ -0,0 +1,32 @@
|
||||
import colors from 'colors/safe.js';
|
||||
import * as diff from 'diff';
|
||||
import * as fs from 'fs';
|
||||
|
||||
import * as toc from '../tool/toc.js';
|
||||
|
||||
toc.files.forEach(file => {
|
||||
const markdown = fs.readFileSync(file).toString();
|
||||
console.log('Reading: ' + file);
|
||||
|
||||
const currentToc = toc.getCurrent(markdown);
|
||||
if (currentToc === null) {
|
||||
console.log(colors.yellow("File doesn't have a table of contents"));
|
||||
return;
|
||||
}
|
||||
|
||||
const generatedToc = toc.generate(markdown);
|
||||
if (currentToc === generatedToc) return;
|
||||
|
||||
const tocDiff = diff.diffLines(currentToc, generatedToc);
|
||||
tocDiff.forEach(part => {
|
||||
const color = part.added
|
||||
? colors.green
|
||||
: part.removed
|
||||
? colors.red
|
||||
: colors.grey;
|
||||
process.stderr.write(color(part.value));
|
||||
});
|
||||
process.stderr.write('\n');
|
||||
|
||||
process.exitCode = 1;
|
||||
});
|
@ -1,30 +0,0 @@
|
||||
import colors from 'colors/safe.js'
|
||||
import diff from 'diff'
|
||||
import * as fs from 'fs'
|
||||
|
||||
import * as toc from '../tool/src/toc.mjs'
|
||||
|
||||
toc.files.forEach(function (file) {
|
||||
var markdown = fs.readFileSync(file).toString()
|
||||
console.log('Reading: ' + file)
|
||||
|
||||
var currentToc = toc.getCurrent(markdown)
|
||||
if (currentToc == null) {
|
||||
console.log(colors.yellow("File doesn't have a table of contents"))
|
||||
return
|
||||
}
|
||||
|
||||
var generatedToc = toc.generate(markdown)
|
||||
if (currentToc === generatedToc) return
|
||||
|
||||
var tocDiff = diff.diffLines(currentToc, generatedToc)
|
||||
tocDiff.forEach(function (part) {
|
||||
var color = part.added
|
||||
? colors.green
|
||||
: (part.removed ? colors.red : colors.grey)
|
||||
process.stderr.write(color(part.value))
|
||||
})
|
||||
process.stderr.write('\n')
|
||||
|
||||
process.exitCode = 1
|
||||
})
|
@ -1,36 +0,0 @@
|
||||
import glob from 'glob'
|
||||
import markdownToc from 'markdown-toc'
|
||||
|
||||
/// Files that may contain tables of contents.
|
||||
export const files = glob.sync('**/*.md', {
|
||||
ignore: ['node_modules/**/*.md', '**/*.changes.md']
|
||||
})
|
||||
|
||||
// Returns the index of the first `*` in `markdown`'s table of contents, or
|
||||
// `null` if `markdown` doesn't include a table of contents.
|
||||
function findStartIndex (markdown) {
|
||||
var match = markdown.match(/\n## Table of Contents\n/)
|
||||
if (match == null) return null
|
||||
return match.index + match[0].length - 1
|
||||
}
|
||||
|
||||
// Returns the table of contents in `markdown` if it contains one, or `null`
|
||||
// otherwise.
|
||||
export function getCurrent (markdown) {
|
||||
var startIndex = findStartIndex(markdown)
|
||||
if (startIndex == null) return null
|
||||
|
||||
var endIndex = markdown.indexOf('## ', startIndex) - 1
|
||||
if (endIndex < 0) return null
|
||||
|
||||
return markdown.substring(startIndex, endIndex).trim()
|
||||
}
|
||||
|
||||
/// Returns the expected table of contents for `markdown`.
|
||||
export function generate (markdown) {
|
||||
return markdownToc(markdown, {
|
||||
filter: (string, _, __) => string.indexOf('Table of Contents') === -1,
|
||||
firsth1: false,
|
||||
bullets: '*'
|
||||
}).content
|
||||
}
|
38
tool/toc.js
Normal file
38
tool/toc.js
Normal file
@ -0,0 +1,38 @@
|
||||
import glob from 'glob';
|
||||
import markdownToc from 'markdown-toc';
|
||||
|
||||
/** Files that may contain tables of contents. */
|
||||
export const files = glob.sync('**/*.md', {
|
||||
ignore: ['node_modules/**/*.md', '**/*.changes.md'],
|
||||
});
|
||||
|
||||
// Returns the index of the first `*` in `markdown`'s table of contents, or
|
||||
// `null` if `markdown` doesn't include a table of contents.
|
||||
function findStartIndex(markdown) {
|
||||
const match = markdown.match(/\n## Table of Contents\n/);
|
||||
if (match === null || match.index === undefined) return null;
|
||||
return match.index + match[0].length - 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the table of contents in `markdown` if it contains one, or `null`
|
||||
* otherwise.
|
||||
*/
|
||||
export function getCurrent(markdown) {
|
||||
const startIndex = findStartIndex(markdown);
|
||||
if (startIndex === null) return null;
|
||||
|
||||
const endIndex = markdown.indexOf('## ', startIndex) - 1;
|
||||
if (endIndex < 0) return null;
|
||||
|
||||
return markdown.substring(startIndex, endIndex).trim();
|
||||
}
|
||||
|
||||
/** Returns the expected table of contents for `markdown`. */
|
||||
export function generate(markdown) {
|
||||
return markdownToc(markdown, {
|
||||
filter: contents => contents.indexOf('Table of Contents') === -1,
|
||||
firsth1: false,
|
||||
bullets: '*',
|
||||
}).content;
|
||||
}
|
32
tool/update-toc.js
Normal file
32
tool/update-toc.js
Normal file
@ -0,0 +1,32 @@
|
||||
import * as fs from 'fs';
|
||||
|
||||
import * as toc from './toc.js';
|
||||
|
||||
toc.files.forEach(file => {
|
||||
const markdown = fs.readFileSync(file).toString();
|
||||
|
||||
const currentToc = toc.getCurrent(markdown);
|
||||
if (currentToc === null) {
|
||||
const match = markdown.match('## Table of Contents\n\n');
|
||||
if (!match || !match.index) return;
|
||||
|
||||
const tocLocation = match.index + match[0].length;
|
||||
|
||||
// If there's an empty TOC, fill it in.
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
markdown.substring(0, tocLocation) +
|
||||
toc.generate(markdown) +
|
||||
'\n\n' +
|
||||
markdown.substring(tocLocation)
|
||||
);
|
||||
console.log(`Added TOC to ${file}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const generatedToc = toc.generate(markdown);
|
||||
if (currentToc === generatedToc) return;
|
||||
|
||||
fs.writeFileSync(file, markdown.replace(currentToc, generatedToc));
|
||||
console.log(`Updated TOC in ${file}`);
|
||||
});
|
@ -1,29 +0,0 @@
|
||||
import * as toc from './src/toc.mjs'
|
||||
import * as fs from 'fs'
|
||||
|
||||
toc.files.forEach(function (file) {
|
||||
var markdown = fs.readFileSync(file).toString()
|
||||
|
||||
var currentToc = toc.getCurrent(markdown)
|
||||
if (currentToc == null) {
|
||||
var match = markdown.match('## Table of Contents\n\n')
|
||||
if (!match) return
|
||||
|
||||
var tocLocation = match.index + match[0].length
|
||||
|
||||
// If there's an empty TOC, fill it in.
|
||||
fs.writeFileSync(
|
||||
file,
|
||||
markdown.substring(0, tocLocation) +
|
||||
toc.generate(markdown) + '\n\n' +
|
||||
markdown.substring(tocLocation))
|
||||
console.log(`Added TOC to ${file}`)
|
||||
return
|
||||
}
|
||||
|
||||
var generatedToc = toc.generate(markdown)
|
||||
if (currentToc === generatedToc) return
|
||||
|
||||
fs.writeFileSync(file, markdown.replace(currentToc, generatedToc))
|
||||
console.log(`Updated TOC in ${file}`)
|
||||
})
|
@ -1,10 +1,8 @@
|
||||
{
|
||||
"extends": "./node_modules/gts/tsconfig-google.json",
|
||||
"compilerOptions": {"rootDir": "."},
|
||||
"include": [
|
||||
"index.d.ts",
|
||||
"spec/**/*.ts",
|
||||
"proposal/**/*.ts",
|
||||
"accepted/**/*.ts"
|
||||
]
|
||||
"compilerOptions": {
|
||||
"allowJs": true,
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["accepted/**/*.ts", "proposal/**/*.ts", "spec/**/*.ts"]
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user