mirror of
https://github.com/sass/sass.git
synced 2024-09-21 02:27:30 +00:00
Rate limit Markdown link checks (#3112)
This commit is contained in:
parent
0c11d8576f
commit
7cd83369e9
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@ -1,7 +1,7 @@
|
|||||||
name: CI
|
name: CI
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push: {branches: [master, 'feature.*']}
|
push: {branches: [main, 'feature.*']}
|
||||||
pull_request:
|
pull_request:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
@ -131,7 +131,7 @@ accepted.
|
|||||||
See [Plain CSS `min()` and `max()`][min-max background] for a good example of
|
See [Plain CSS `min()` and `max()`][min-max background] for a good example of
|
||||||
a Background section.
|
a Background section.
|
||||||
|
|
||||||
[min-max background]: https://github.com/sass/sass/blob/master/accepted/min-max.md#background
|
[min-max background]: https://github.com/sass/sass/blob/main/accepted/min-max.md#background
|
||||||
|
|
||||||
* **Summary**
|
* **Summary**
|
||||||
|
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
<h1><img width="200px" alt="Sass" src="https://rawgit.com/sass/sass-site/master/source/assets/img/logos/logo.svg" /></h1>
|
<h1><img width="200px" alt="Sass" src="https://rawgit.com/sass/sass-site/main/source/assets/img/logos/logo.svg" /></h1>
|
||||||
|
|
||||||
[![@SassCSS on Twitter](https://img.shields.io/twitter/follow/SassCSS?label=%40SassCSS&style=social)](https://twitter.com/SassCSS)
|
[![@SassCSS on Twitter](https://img.shields.io/twitter/follow/SassCSS?label=%40SassCSS&style=social)](https://twitter.com/SassCSS)
|
||||||
|
|
||||||
@ -96,9 +96,9 @@ This repository isn't an implementation of Sass. Those live in
|
|||||||
* [`accepted/`][], which contains proposals that have been accepted and are
|
* [`accepted/`][], which contains proposals that have been accepted and are
|
||||||
either implemented or in the process of being implemented.
|
either implemented or in the process of being implemented.
|
||||||
|
|
||||||
[`spec/`]: https://github.com/sass/sass/tree/master/spec
|
[`spec/`]: https://github.com/sass/sass/tree/main/spec
|
||||||
[`proposal/`]: https://github.com/sass/sass/tree/master/proposal
|
[`proposal/`]: https://github.com/sass/sass/tree/main/proposal
|
||||||
[`accepted/`]: https://github.com/sass/sass/tree/master/accepted
|
[`accepted/`]: https://github.com/sass/sass/tree/main/accepted
|
||||||
|
|
||||||
Note that this doesn't contain a full specification of Sass. Instead, feature
|
Note that this doesn't contain a full specification of Sass. Instead, feature
|
||||||
specifications are written as needed when a new feature is being designed or
|
specifications are written as needed when a new feature is being designed or
|
||||||
|
3399
package-lock.json
generated
3399
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
18
package.json
18
package.json
@ -1,13 +1,20 @@
|
|||||||
{
|
{
|
||||||
"name": "sass",
|
"name": "sass",
|
||||||
"//": [
|
"//": [
|
||||||
"This is used to track dependencies for the Travis CI tests run on this ",
|
"This is used to track dependencies for the Github Actions tests run on ",
|
||||||
"repository. For the official Sass npm package, see ",
|
"this repository. For the official Sass npm package, see ",
|
||||||
"https://npmjs.org/package/sass"
|
"https://npmjs.org/package/sass"
|
||||||
],
|
],
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=14.0.0 <17.0.0"
|
"node": ">=14.0.0 <17.0.0"
|
||||||
},
|
},
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"link-check": "node test/link-check.js",
|
||||||
|
"toc-check": "node test/toc-check.js",
|
||||||
|
"fix": "gts fix",
|
||||||
|
"test": "gts lint && tsc --noEmit && npm run toc-check && npm run link-check"
|
||||||
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"colors": "^1.3.3",
|
"colors": "^1.3.3",
|
||||||
"diff": "^4.0.1",
|
"diff": "^4.0.1",
|
||||||
@ -20,13 +27,6 @@
|
|||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^14.11.2",
|
"@types/node": "^14.11.2",
|
||||||
"gts": "^3.1.0",
|
"gts": "^3.1.0",
|
||||||
"standard": "^14.3.4",
|
|
||||||
"typescript": "^4.0.3"
|
"typescript": "^4.0.3"
|
||||||
},
|
|
||||||
"scripts": {
|
|
||||||
"link-check": "node test/link-check.mjs",
|
|
||||||
"toc-check": "node test/toc-check.mjs",
|
|
||||||
"fix": "gts fix && standard --fix",
|
|
||||||
"test": "gts lint && standard && tsc --noEmit && npm run toc-check && npm run link-check"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
99
test/link-check.js
Normal file
99
test/link-check.js
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
import colors from 'colors/safe.js';
|
||||||
|
import * as fs from 'fs';
|
||||||
|
import glob from 'glob';
|
||||||
|
import markdownLinkCheck from 'markdown-link-check';
|
||||||
|
import markdownToc from 'markdown-toc';
|
||||||
|
import * as path from 'path';
|
||||||
|
import {fileURLToPath, pathToFileURL, URL} from 'url';
|
||||||
|
|
||||||
|
const files = glob.sync('**/*.md', {
|
||||||
|
ignore: ['node_modules/**/*.md'],
|
||||||
|
});
|
||||||
|
|
||||||
|
const tocCache = new Map();
|
||||||
|
|
||||||
|
function getToc(file) {
|
||||||
|
file = path.normalize(file);
|
||||||
|
let toc = tocCache.get(file);
|
||||||
|
if (toc === undefined) {
|
||||||
|
toc = markdownToc(fs.readFileSync(file).toString(), {}).content;
|
||||||
|
tocCache.set(file, toc);
|
||||||
|
}
|
||||||
|
return toc;
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyLinkCheckResults(file, results) {
|
||||||
|
const toc = getToc(file);
|
||||||
|
|
||||||
|
for (const result of results) {
|
||||||
|
const url = new URL(result.link, pathToFileURL(file));
|
||||||
|
|
||||||
|
// A link to another file.
|
||||||
|
if (url.protocol === 'file:' && !result.link.match(/ \(.*\)$/)) {
|
||||||
|
const target = fileURLToPath(url);
|
||||||
|
if (!fs.existsSync(target)) throw Error(`Missing file: ${result.link}`);
|
||||||
|
if (url.hash === '') return;
|
||||||
|
if (getToc(target).includes(url.hash)) return;
|
||||||
|
throw Error(`Dead: ${result.link}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A link to a section within this file.
|
||||||
|
if (result.link.match(/^#/)) {
|
||||||
|
if (toc.includes(result.link)) return;
|
||||||
|
throw Error(`Dead: ${result.link}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A link to an external website.
|
||||||
|
switch (result.status) {
|
||||||
|
case 'dead':
|
||||||
|
if (result.statusCode === 500) {
|
||||||
|
console.log(colors.yellow(`Server error on target: ${result.link}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw Error(`Dead: ${result.link}`);
|
||||||
|
case 'error':
|
||||||
|
throw Error(`Error: ${result.link}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function runLinkCheck(file, {rateLimit}) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
setTimeout(check, rateLimit);
|
||||||
|
|
||||||
|
function check() {
|
||||||
|
markdownLinkCheck(
|
||||||
|
fs.readFileSync(file).toString(),
|
||||||
|
{
|
||||||
|
baseUrl: '',
|
||||||
|
// If Github rate limit is reached, wait 60s and try again.
|
||||||
|
retryOn429: true,
|
||||||
|
},
|
||||||
|
(error, results) => {
|
||||||
|
if (error) {
|
||||||
|
reject(error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
verifyLinkCheckResults(file, results);
|
||||||
|
resolve();
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
for (const file of files) {
|
||||||
|
console.log('Checking links in ' + file);
|
||||||
|
try {
|
||||||
|
await runLinkCheck(file, {rateLimit: 500});
|
||||||
|
} catch (error) {
|
||||||
|
console.log(colors.red(error.message));
|
||||||
|
process.exitCode = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})();
|
@ -1,83 +0,0 @@
|
|||||||
import colors from 'colors/safe.js'
|
|
||||||
import * as fs from 'fs'
|
|
||||||
import glob from 'glob'
|
|
||||||
import toc from 'markdown-toc'
|
|
||||||
import * as urlModule from 'url'
|
|
||||||
import markdownLinkCheck from 'markdown-link-check'
|
|
||||||
import * as path from 'path'
|
|
||||||
|
|
||||||
var files = glob.sync('**/*.md', { ignore: ['node_modules/**/*.md'] })
|
|
||||||
|
|
||||||
var tocCache = new Map()
|
|
||||||
|
|
||||||
function getToc (file) {
|
|
||||||
file = path.normalize(file)
|
|
||||||
if (tocCache.has(file)) {
|
|
||||||
return tocCache.get(file)
|
|
||||||
} else {
|
|
||||||
var result = toc(fs.readFileSync(file).toString()).content
|
|
||||||
tocCache.set(file, result)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
files.forEach(function (file) {
|
|
||||||
var markdown = fs.readFileSync(file).toString()
|
|
||||||
|
|
||||||
var dirname = path.dirname(urlModule.fileURLToPath(import.meta.url))
|
|
||||||
markdownLinkCheck(markdown, {
|
|
||||||
retryOn429: true, // Retry if the github rate limit is reached
|
|
||||||
baseUrl: path.basename(dirname) + '/'
|
|
||||||
}, function (err, results) {
|
|
||||||
if (err) {
|
|
||||||
console.error('Error', err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('Reading: ' + file)
|
|
||||||
|
|
||||||
// Get a list of all headers so we can verify intra-document links.
|
|
||||||
var markdownToc = getToc(file)
|
|
||||||
|
|
||||||
results.forEach(function (result) {
|
|
||||||
var url = new URL(result.link, urlModule.pathToFileURL(file))
|
|
||||||
if (url.protocol === 'file:' && !result.link.match(/ \(.*\)$/)) {
|
|
||||||
var target = urlModule.fileURLToPath(url)
|
|
||||||
if (!fs.existsSync(target)) {
|
|
||||||
process.exitCode = 1
|
|
||||||
console.log(colors.red(`Missing file: ${result.link}`))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (url.hash === '') return
|
|
||||||
var toc = getToc(target)
|
|
||||||
|
|
||||||
if (toc.includes(url.hash)) return
|
|
||||||
process.exitCode = 1
|
|
||||||
console.log(colors.red(`Dead: ${result.link}`))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.link.match(/^#/)) {
|
|
||||||
if (markdownToc.includes(result.link)) {
|
|
||||||
result.status = 'alive'
|
|
||||||
} else {
|
|
||||||
result.status = 'dead'
|
|
||||||
result.statusCode = 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.status === 'dead') {
|
|
||||||
if (result.statusCode === 500) {
|
|
||||||
console.log(colors.yellow(`Server error on target: ${result.link}`))
|
|
||||||
} else {
|
|
||||||
process.exitCode = 1
|
|
||||||
console.log(colors.red(`Dead: ${result.link}`))
|
|
||||||
}
|
|
||||||
} else if (result.status === 'error') {
|
|
||||||
process.exitCode = 1
|
|
||||||
console.log(colors.red(`Error: ${result.link}`))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
})
|
|
32
test/toc-check.js
Normal file
32
test/toc-check.js
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
import colors from 'colors/safe.js';
|
||||||
|
import * as diff from 'diff';
|
||||||
|
import * as fs from 'fs';
|
||||||
|
|
||||||
|
import * as toc from '../tool/toc.js';
|
||||||
|
|
||||||
|
toc.files.forEach(file => {
|
||||||
|
const markdown = fs.readFileSync(file).toString();
|
||||||
|
console.log('Reading: ' + file);
|
||||||
|
|
||||||
|
const currentToc = toc.getCurrent(markdown);
|
||||||
|
if (currentToc === null) {
|
||||||
|
console.log(colors.yellow("File doesn't have a table of contents"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const generatedToc = toc.generate(markdown);
|
||||||
|
if (currentToc === generatedToc) return;
|
||||||
|
|
||||||
|
const tocDiff = diff.diffLines(currentToc, generatedToc);
|
||||||
|
tocDiff.forEach(part => {
|
||||||
|
const color = part.added
|
||||||
|
? colors.green
|
||||||
|
: part.removed
|
||||||
|
? colors.red
|
||||||
|
: colors.grey;
|
||||||
|
process.stderr.write(color(part.value));
|
||||||
|
});
|
||||||
|
process.stderr.write('\n');
|
||||||
|
|
||||||
|
process.exitCode = 1;
|
||||||
|
});
|
@ -1,30 +0,0 @@
|
|||||||
import colors from 'colors/safe.js'
|
|
||||||
import diff from 'diff'
|
|
||||||
import * as fs from 'fs'
|
|
||||||
|
|
||||||
import * as toc from '../tool/src/toc.mjs'
|
|
||||||
|
|
||||||
toc.files.forEach(function (file) {
|
|
||||||
var markdown = fs.readFileSync(file).toString()
|
|
||||||
console.log('Reading: ' + file)
|
|
||||||
|
|
||||||
var currentToc = toc.getCurrent(markdown)
|
|
||||||
if (currentToc == null) {
|
|
||||||
console.log(colors.yellow("File doesn't have a table of contents"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var generatedToc = toc.generate(markdown)
|
|
||||||
if (currentToc === generatedToc) return
|
|
||||||
|
|
||||||
var tocDiff = diff.diffLines(currentToc, generatedToc)
|
|
||||||
tocDiff.forEach(function (part) {
|
|
||||||
var color = part.added
|
|
||||||
? colors.green
|
|
||||||
: (part.removed ? colors.red : colors.grey)
|
|
||||||
process.stderr.write(color(part.value))
|
|
||||||
})
|
|
||||||
process.stderr.write('\n')
|
|
||||||
|
|
||||||
process.exitCode = 1
|
|
||||||
})
|
|
@ -1,36 +0,0 @@
|
|||||||
import glob from 'glob'
|
|
||||||
import markdownToc from 'markdown-toc'
|
|
||||||
|
|
||||||
/// Files that may contain tables of contents.
|
|
||||||
export const files = glob.sync('**/*.md', {
|
|
||||||
ignore: ['node_modules/**/*.md', '**/*.changes.md']
|
|
||||||
})
|
|
||||||
|
|
||||||
// Returns the index of the first `*` in `markdown`'s table of contents, or
|
|
||||||
// `null` if `markdown` doesn't include a table of contents.
|
|
||||||
function findStartIndex (markdown) {
|
|
||||||
var match = markdown.match(/\n## Table of Contents\n/)
|
|
||||||
if (match == null) return null
|
|
||||||
return match.index + match[0].length - 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returns the table of contents in `markdown` if it contains one, or `null`
|
|
||||||
// otherwise.
|
|
||||||
export function getCurrent (markdown) {
|
|
||||||
var startIndex = findStartIndex(markdown)
|
|
||||||
if (startIndex == null) return null
|
|
||||||
|
|
||||||
var endIndex = markdown.indexOf('## ', startIndex) - 1
|
|
||||||
if (endIndex < 0) return null
|
|
||||||
|
|
||||||
return markdown.substring(startIndex, endIndex).trim()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns the expected table of contents for `markdown`.
|
|
||||||
export function generate (markdown) {
|
|
||||||
return markdownToc(markdown, {
|
|
||||||
filter: (string, _, __) => string.indexOf('Table of Contents') === -1,
|
|
||||||
firsth1: false,
|
|
||||||
bullets: '*'
|
|
||||||
}).content
|
|
||||||
}
|
|
38
tool/toc.js
Normal file
38
tool/toc.js
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
import glob from 'glob';
|
||||||
|
import markdownToc from 'markdown-toc';
|
||||||
|
|
||||||
|
/** Files that may contain tables of contents. */
|
||||||
|
export const files = glob.sync('**/*.md', {
|
||||||
|
ignore: ['node_modules/**/*.md', '**/*.changes.md'],
|
||||||
|
});
|
||||||
|
|
||||||
|
// Returns the index of the first `*` in `markdown`'s table of contents, or
|
||||||
|
// `null` if `markdown` doesn't include a table of contents.
|
||||||
|
function findStartIndex(markdown) {
|
||||||
|
const match = markdown.match(/\n## Table of Contents\n/);
|
||||||
|
if (match === null || match.index === undefined) return null;
|
||||||
|
return match.index + match[0].length - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the table of contents in `markdown` if it contains one, or `null`
|
||||||
|
* otherwise.
|
||||||
|
*/
|
||||||
|
export function getCurrent(markdown) {
|
||||||
|
const startIndex = findStartIndex(markdown);
|
||||||
|
if (startIndex === null) return null;
|
||||||
|
|
||||||
|
const endIndex = markdown.indexOf('## ', startIndex) - 1;
|
||||||
|
if (endIndex < 0) return null;
|
||||||
|
|
||||||
|
return markdown.substring(startIndex, endIndex).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Returns the expected table of contents for `markdown`. */
|
||||||
|
export function generate(markdown) {
|
||||||
|
return markdownToc(markdown, {
|
||||||
|
filter: contents => contents.indexOf('Table of Contents') === -1,
|
||||||
|
firsth1: false,
|
||||||
|
bullets: '*',
|
||||||
|
}).content;
|
||||||
|
}
|
32
tool/update-toc.js
Normal file
32
tool/update-toc.js
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
import * as fs from 'fs';
|
||||||
|
|
||||||
|
import * as toc from './toc.js';
|
||||||
|
|
||||||
|
toc.files.forEach(file => {
|
||||||
|
const markdown = fs.readFileSync(file).toString();
|
||||||
|
|
||||||
|
const currentToc = toc.getCurrent(markdown);
|
||||||
|
if (currentToc === null) {
|
||||||
|
const match = markdown.match('## Table of Contents\n\n');
|
||||||
|
if (!match || !match.index) return;
|
||||||
|
|
||||||
|
const tocLocation = match.index + match[0].length;
|
||||||
|
|
||||||
|
// If there's an empty TOC, fill it in.
|
||||||
|
fs.writeFileSync(
|
||||||
|
file,
|
||||||
|
markdown.substring(0, tocLocation) +
|
||||||
|
toc.generate(markdown) +
|
||||||
|
'\n\n' +
|
||||||
|
markdown.substring(tocLocation)
|
||||||
|
);
|
||||||
|
console.log(`Added TOC to ${file}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const generatedToc = toc.generate(markdown);
|
||||||
|
if (currentToc === generatedToc) return;
|
||||||
|
|
||||||
|
fs.writeFileSync(file, markdown.replace(currentToc, generatedToc));
|
||||||
|
console.log(`Updated TOC in ${file}`);
|
||||||
|
});
|
@ -1,29 +0,0 @@
|
|||||||
import * as toc from './src/toc.mjs'
|
|
||||||
import * as fs from 'fs'
|
|
||||||
|
|
||||||
toc.files.forEach(function (file) {
|
|
||||||
var markdown = fs.readFileSync(file).toString()
|
|
||||||
|
|
||||||
var currentToc = toc.getCurrent(markdown)
|
|
||||||
if (currentToc == null) {
|
|
||||||
var match = markdown.match('## Table of Contents\n\n')
|
|
||||||
if (!match) return
|
|
||||||
|
|
||||||
var tocLocation = match.index + match[0].length
|
|
||||||
|
|
||||||
// If there's an empty TOC, fill it in.
|
|
||||||
fs.writeFileSync(
|
|
||||||
file,
|
|
||||||
markdown.substring(0, tocLocation) +
|
|
||||||
toc.generate(markdown) + '\n\n' +
|
|
||||||
markdown.substring(tocLocation))
|
|
||||||
console.log(`Added TOC to ${file}`)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var generatedToc = toc.generate(markdown)
|
|
||||||
if (currentToc === generatedToc) return
|
|
||||||
|
|
||||||
fs.writeFileSync(file, markdown.replace(currentToc, generatedToc))
|
|
||||||
console.log(`Updated TOC in ${file}`)
|
|
||||||
})
|
|
@ -1,10 +1,8 @@
|
|||||||
{
|
{
|
||||||
"extends": "./node_modules/gts/tsconfig-google.json",
|
"extends": "./node_modules/gts/tsconfig-google.json",
|
||||||
"compilerOptions": {"rootDir": "."},
|
"compilerOptions": {
|
||||||
"include": [
|
"allowJs": true,
|
||||||
"index.d.ts",
|
"rootDir": "."
|
||||||
"spec/**/*.ts",
|
},
|
||||||
"proposal/**/*.ts",
|
"include": ["accepted/**/*.ts", "proposal/**/*.ts", "spec/**/*.ts"]
|
||||||
"accepted/**/*.ts"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user