kmx git

Commit 44456b21d0bc6aeea84bffe7622b43ffb69f9591

2021-08-16T20:08:13
Added benchmark suite (#2153)
diff --git a/.eslintrc.js b/.eslintrc.js
index b3a54bb..5002ad9 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -188,10 +188,11 @@ module.exports = {
 			}
 		},
 		{
-			// Gulp and Danger
+			// Gulp, Danger, and benchmark
 			files: [
 				'gulpfile.js/**',
-				'dangerfile.js'
+				'dangerfile.js',
+				'benchmark/**',
 			],
 			env: {
 				es6: true,
diff --git a/.gitignore b/.gitignore
index 019ed62..f3e4015 100755
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,6 @@ node_modules
 .idea/
 .DS_Store
 .eslintcache
+
+benchmark/remotes/
+benchmark/downloads/
diff --git a/.npmignore b/.npmignore
index 9710615..aa4fd97 100644
--- a/.npmignore
+++ b/.npmignore
@@ -5,6 +5,7 @@ hide-*.js
 .DS_Store
 CNAME
 .github/
+benchmark/
 assets/
 docs/
 examples/
diff --git a/benchmark.html b/benchmark.html
new file mode 100644
index 0000000..0b19f8e
--- /dev/null
+++ b/benchmark.html
@@ -0,0 +1,144 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+
+<meta charset="utf-8" />
+<link rel="icon" href="assets/favicon.png" />
+<title>Benchmark ▲ Prism</title>
+<link rel="stylesheet" href="assets/style.css" />
+<link rel="stylesheet" href="themes/prism.css" data-noprefix />
+<script src="assets/vendor/prefixfree.min.js"></script>
+
+<script>var _gaq = [['_setAccount', 'UA-33746269-1'], ['_trackPageview']];</script>
+<script src="https://www.google-analytics.com/ga.js" async></script>
+</head>
+<body class="language-javascript">
+
+<header>
+	<div class="intro" data-src="assets/templates/header-main.html" data-type="text/html"></div>
+
+	<h2>Benchmark</h2>
+	<p>Prism has a benchmark suite which can be run and extended similar to the test suite.</p>
+</header>
+
+<section id="running-a-benchmark">
+	<h1>Running a benchmark</h1>
+
+	<pre><code class="language-bash">npm run benchmark</code></pre>
+
+	<p>By default, the benchmark will be run for the current local version of your repository (the one which is currently checkout) and the <a href="https://github.com/PrismJS/prism/tree/master">PrismJS master branch</a>.</p>
+
+	<p>All <code>options</code> in <code>benchmark/config.json</code> can be changed by either directly editing the file or by passing arguments to the run command. I.e. you can overwrite the default <code>maxTime</code> value with 10s by running the following command:</p>
+
+	<pre><code class="language-bash">npm run benchmark -- --maxTime=10</code></pre>
+
+	<section id="running-a-benchmark-for-specific-languages">
+		<h2>Running a benchmark for specific languages</h2>
+
+		<p>To run the tests only for a certain set of languages, you can use the <code>language</code> option:</p>
+		<pre><code class="language-bash">npm run benchmark -- --language=javascript,markup</code></pre>
+	</section>
+</section>
+
+<section id="remotes">
+	<h1>Remotes</h1>
+
+	<p>Remotes all you to compare different branches from different repos or the same repo. Repos can be the PrismJS repo or any your fork.</p>
+
+	<p>All remotes are specified under the <code>remotes</code> section in <code>benchmark/config.json</code>. To add a new remote, add an object with the <code>repo</code> and <code>branch</code> properties to the array. Note: if the branch property is not specified, the <code>master</code> branch will be used. <br>
+		Example:</p>
+
+	<pre><code class="language-javascript">{
+	repo: 'https://github.com/MyUserName/prism.git',
+	branch: 'feature-1'
+}</code></pre>
+
+	<p>To remove a remote, simply remove (delete or comment out) its entry in the <code>remotes</code> array.</p>
+</section>
+
+<section id="cases">
+	<h1>Cases</h1>
+
+	<p>A case is a collection of files where each file will be benchmarked with all candidates (local + remotes) and a specific language.</p>
+
+	<p>The language of a case is determined by its key in the <code>cases</code> object in <code>benchmark/config.json</code> where the key has to have the same format as the directory names in <code class="language-text">tests/languages/</code>. Example:</p>
+	<pre><code class="language-javascript">cases: {
+	'css!+css-extras': ...
+}</code></pre>
+
+	<p>The files of a case can be specified by:</p>
+
+	<ul>
+		<li>
+			<p>Specifying the URI of files. A URI is either an HTTPS URL or a file path relative to <code>./benchmark/</code>.</p>
+			<pre><code class="language-javascript">cases: {
+	'css': {
+		files: [
+			'style.css',
+			'https://foo.com/main.css'
+		]
+	}
+}</code></pre>
+		</li>
+		<li>
+			<p>Using <code>extends</code> to copy all files from another case.</p>
+			<pre><code class="language-javascript">cases: {
+	'css': { files: [ 'style.css' ] },
+	'css!+css-extras': {
+		extends: 'css'
+	}
+}</code></pre>
+		</li>
+	</ul>
+</section>
+
+<section id="output-explained">
+	<h1>Output explained</h1>
+
+	<p>The output of a benchmark might look like this:</p>
+
+	<pre><code class="language-none">Found 1 cases with 2 files in total.
+Test 3 candidates on tokenize
+Estimated duration: 1m 0s</code></pre>
+
+	<p>The first few lines give some information on the benchmark which is about to be run. This includes the number of cases (here 1), the total number of files in all cases (here 2), the number of candidates (here 3), the test function (here <code>tokenize</code>), and a time estimate for how long the benchmark will take (here 1 minute).</p>
+
+	<p>What follows are the results for all cases and their files:</p>
+
+	<pre><code class="language-none">json
+
+  components.json (25 kB)
+  | local                         5.45ms ± 13% 138smp
+  | PrismJS@master                4.92ms ±  2% 145smp
+  | RunDevelopment@greedy-fix     5.62ms ±  4% 128smp
+  package-lock.json (189 kB)
+  | local                       117.75ms ± 27% 27smp
+  | PrismJS@master              121.40ms ± 32% 29smp
+  | RunDevelopment@greedy-fix   190.79ms ± 41% 20smp</code></pre>
+
+	<p>A case starts with its key (here <code>json</code>) and is followed by all of its files (here <code>components.json</code> and <code>package-lock.json</code>). Under each file, the results for local and all remotes are shown. To explain the meaning of the values, let's pick a single line:</p>
+
+	<code class="language-none">PrismJS@master              121.40ms ± 32% 29smp</code>
+
+	<p>First comes the name of the remote (here PrismJS@master) followed by the mean of all samples, the relative margin of error, and the number of samples.</p>
+
+	<p>The last part of the output is a small summary of all cases which simply counts how many times a candidate has been the best or worst for a file.</p>
+
+	<pre><code class="language-none">summary
+                             best  worst
+  local                         1      1
+  PrismJS@master                0      1
+  RunDevelopment@greedy-fix     1      0</code></pre>
+</section>
+
+
+<footer data-src="assets/templates/footer.html" data-type="text/html"></footer>
+
+<script src="assets/vendor/utopia.js"></script>
+<script src="prism.js"></script>
+<script src="components/prism-bash.js"></script>
+<script src="components.js"></script>
+<script src="assets/code.js"></script>
+
+</body>
+</html>
diff --git a/benchmark/benchmark.js b/benchmark/benchmark.js
new file mode 100644
index 0000000..ebbfac1
--- /dev/null
+++ b/benchmark/benchmark.js
@@ -0,0 +1,454 @@
+// @ts-check
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { argv } = require('yargs');
+const fetch = require('node-fetch').default;
+const Benchmark = require('benchmark');
+const simpleGit = require('simple-git/promise');
+const { parseLanguageNames } = require('../tests/helper/test-case');
+
+
+/**
+ * @param {import("./config").Config} config
+ */
+async function runBenchmark(config) {
+	const cases = await getCases(config);
+	const candidates = await getCandidates(config);
+	const maxCandidateNameLength = candidates.reduce((a, c) => Math.max(a, c.name.length), 0);
+
+	const totalNumberOfCaseFiles = cases.reduce((a, c) => a + c.files.length, 0);
+	console.log(`Found ${cases.length} cases with ${totalNumberOfCaseFiles} files in total.`);
+	console.log(`Test ${candidates.length} candidates with Prism.${config.options.testFunction}`);
+	const estimate = candidates.length * totalNumberOfCaseFiles * config.options.maxTime;
+	console.log(`Estimated duration: ${Math.floor(estimate / 60)}m ${Math.floor(estimate % 60)}s`);
+
+	/**
+	 * @type {Summary[]}
+	 *
+	 * @typedef {{ best: number; worst: number, relative: number[], avgRelative?: number }} Summary
+	 */
+	const totalSummary = Array.from({ length: candidates.length }, () => ({ best: 0, worst: 0, relative: [] }));
+
+	for (const $case of cases) {
+
+		console.log();
+		console.log(`\x1b[90m${'-'.repeat(60)}\x1b[0m`);
+		console.log();
+		if ($case.id !== $case.language) {
+			console.log(`${$case.id} (${$case.language})`);
+		} else {
+			console.log($case.id);
+		}
+		console.log();
+
+		// prepare candidates
+		const warmupCode = await fs.promises.readFile($case.files[0].path, 'utf8');
+		/** @type {[string, (code: string) => void][]} */
+		const candidateFunctions = candidates.map(({ name, setup }) => {
+			const fn = setup($case.language, $case.languages);
+			fn(warmupCode); // warmup
+			return [name, fn];
+		});
+
+
+		// bench all files
+		for (const caseFile of $case.files) {
+			console.log(`  ${caseFile.uri} \x1b[90m(${Math.round(caseFile.size / 1024)} kB)\x1b[0m`);
+
+			const code = await fs.promises.readFile(caseFile.path, 'utf8');
+
+			const results = measureCandidates(candidateFunctions.map(([name, fn]) => [name, () => fn(code)]), {
+				maxTime: config.options.maxTime,
+				minSamples: 1,
+				delay: 0,
+			});
+
+			const min = results.reduce((a, c) => Math.min(a, c.stats.mean), Infinity);
+			const max = results.reduce((a, c) => Math.max(a, c.stats.mean), -Infinity);
+			const minIndex = results.findIndex(x => x.stats.mean === min);
+			const maxIndex = results.findIndex(x => x.stats.mean === max);
+
+			totalSummary[minIndex].best++;
+			totalSummary[maxIndex].worst++;
+
+			const best = getBest(results);
+			const worst = getWorst(results);
+
+			results.forEach((r, index) => {
+				const name = r.name.padEnd(maxCandidateNameLength, ' ');
+				const mean = (r.stats.mean * 1000).toFixed(2).padStart(8) + 'ms';
+				const r_moe = (100 * r.stats.moe / r.stats.mean).toFixed(0).padStart(3) + '%';
+				const smp = r.stats.sample.length.toString().padStart(4) + 'smp';
+
+				const relativeMean = r.stats.mean / min;
+				totalSummary[index].relative.push(relativeMean);
+				const relative = relativeMean === 1 ? ' '.repeat(5) : (relativeMean.toFixed(2) + 'x').padStart(5);
+
+				const color = r === best ? '\x1b[32m' : r === worst ? '\x1b[31m' : '\x1b[0m';
+
+				console.log(`  \x1b[90m| ${color}${name} ${mean} ±${r_moe} ${smp} ${relative}\x1b[0m`);
+			});
+		}
+	}
+
+	// total summary
+	console.log();
+	console.log(`\x1b[90m${'-'.repeat(60)}\x1b[0m`);
+	console.log();
+	console.log('summary');
+	console.log(`${' '.repeat(maxCandidateNameLength + 2)}  \x1b[90mbest  worst  relative\x1b[0m`);
+
+	totalSummary.forEach(s => {
+		s.avgRelative = s.relative.reduce((a, c) => a + c, 0) / s.relative.length;
+	});
+	const minAvgRelative = totalSummary.reduce((a, c) => Math.min(a, c.avgRelative), Infinity);
+
+	totalSummary.forEach((s, i) => {
+		const name = candidates[i].name.padEnd(maxCandidateNameLength, ' ');
+		const best = String(s.best).padStart('best'.length);
+		const worst = String(s.worst).padStart('worst'.length);
+		const relative = ((s.avgRelative / minAvgRelative).toFixed(2) + 'x').padStart('relative'.length);
+
+		console.log(`  ${name}  ${best}  ${worst}  ${relative}`);
+	});
+}
+
+function getConfig() {
+	const base = require('./config.js');
+
+	const args = /** @type {Record<string, unknown>} */(argv);
+
+	if (typeof args.testFunction === 'string') {
+		// @ts-ignore
+		base.options.testFunction = args.testFunction;
+	}
+	if (typeof args.maxTime === 'number') {
+		base.options.maxTime = args.maxTime;
+	}
+	if (typeof args.language === 'string') {
+		base.options.language = args.language;
+	}
+	if (typeof args.remotesOnly === 'boolean') {
+		base.options.remotesOnly = args.remotesOnly;
+	}
+
+	return base;
+}
+
+/**
+ * @param {import("./config").Config} config
+ * @returns {Promise<Case[]>}
+ *
+ * @typedef Case
+ * @property {string} id
+ * @property {string} language The main language.
+ * @property {string[]} languages All languages that have to be loaded.
+ * @property {FileInfo[]} files
+ */
+async function getCases(config) {
+	/** @type {Map<string, ReadonlySet<FileInfo>>} */
+	const caseFileCache = new Map();
+
+	/**
+	 * Returns all files of the test case with the given id.
+	 *
+	 * @param {string} id
+	 * @returns {Promise<ReadonlySet<FileInfo>>}
+	 */
+	async function getCaseFiles(id) {
+		if (caseFileCache.has(id)) {
+			return caseFileCache.get(id);
+		}
+
+		const caseEntry = config.cases[id];
+		if (!caseEntry) {
+			throw new Error(`Unknown case "${id}"`);
+		}
+
+		/** @type {Set<FileInfo>} */
+		const files = new Set();
+		caseFileCache.set(id, files);
+
+		await Promise.all(toArray(caseEntry.files).map(async uri => {
+			files.add(await getFileInfo(uri));
+		}));
+		for (const extendId of toArray(caseEntry.extends)) {
+			(await getCaseFiles(extendId)).forEach(info => files.add(info));
+		}
+
+		return files;
+	}
+
+	/**
+	 * Returns whether the case is enabled by the options provided by the user.
+	 *
+	 * @param {string[]} languages
+	 * @returns {boolean}
+	 */
+	function isEnabled(languages) {
+		if (config.options.language) {
+			// test whether the given languages contain any of the required languages
+			const required = new Set(config.options.language.split(/,/g).filter(Boolean));
+			return languages.some(l => required.has(l));
+		}
+
+		return true;
+	}
+
+	/** @type {Case[]} */
+	const cases = [];
+	for (const id of Object.keys(config.cases)) {
+		const parsed = parseLanguageNames(id);
+
+		if (!isEnabled(parsed.languages)) {
+			continue;
+		}
+
+		cases.push({
+			id,
+			language: parsed.mainLanguage,
+			languages: parsed.languages,
+			files: [...await getCaseFiles(id)].sort((a, b) => a.uri.localeCompare(b.uri)),
+		});
+	}
+
+	cases.sort((a, b) => a.id.localeCompare(b.id));
+
+	return cases;
+}
+
+/** @type {Map<string, Promise<FileInfo>>} */
+const fileInfoCache = new Map();
+/**
+ * Returns the path and other information for the given file identifier.
+ *
+ * @param {string} uri
+ * @returns {Promise<FileInfo>}
+ *
+ * @typedef {{ uri: string, path: string, size: number }} FileInfo
+ */
+function getFileInfo(uri) {
+	let info = fileInfoCache.get(uri);
+	if (info === undefined) {
+		info = getFileInfoUncached(uri);
+		fileInfoCache.set(uri, info);
+	}
+	return info;
+}
+/**
+ * @param {string} uri
+ * @returns {Promise<FileInfo>}
+ */
+async function getFileInfoUncached(uri) {
+	const p = await getFilePath(uri);
+	const stat = await fs.promises.stat(p);
+	if (stat.isFile()) {
+		return {
+			uri,
+			path: p,
+			size: stat.size
+		};
+	} else {
+		throw new Error(`Unknown file "${uri}"`);
+	}
+}
+/**
+ * Returns the local path of the given file identifier.
+ *
+ * @param {string} uri
+ * @returns {Promise<string>}
+ */
+async function getFilePath(uri) {
+	if (/^https:\/\//.test(uri)) {
+		// it's a URL, so let's download the file (if not downloaded already)
+		const downloadDir = path.join(__dirname, 'downloads');
+		await fs.promises.mkdir(downloadDir, { recursive: true });
+
+		// file path
+		const hash = crypto.createHash('md5').update(uri).digest('hex');
+		const localPath = path.resolve(downloadDir, hash + '-' + /[-\w\.]*$/.exec(uri)[0]);
+
+		if (!fs.existsSync(localPath)) {
+			// download file
+			console.log(`Downloading ${uri}...`);
+			await fs.promises.writeFile(localPath, await fetch(uri).then(r => r.text()), 'utf8');
+		}
+
+		return localPath;
+	}
+
+	// assume that it's a local file
+	return path.resolve(__dirname, uri);
+}
+
+/**
+ * @param {Iterable<[string, () => void]>} candidates
+ * @param {import("benchmark").Options} [options]
+ * @returns {Result[]}
+ *
+ * @typedef {{ name: string, stats: import("benchmark").Stats }} Result
+ */
+function measureCandidates(candidates, options) {
+	const suite = new Benchmark.Suite('temp name');
+
+	for (const [name, fn] of candidates) {
+		suite.add(name, fn, options);
+	}
+
+	/** @type {Result[]} */
+	const results = [];
+
+	suite.on('cycle', event => {
+		results.push({
+			name: event.target.name,
+			stats: event.target.stats
+		});
+	}).run();
+
+	return results;
+}
+
+/**
+ * @param {Result[]} results
+ * @returns {Result | null}
+ */
+function getBest(results) {
+	if (results.length >= 2) {
+		const sorted = [...results].sort((a, b) => a.stats.mean - b.stats.mean);
+		const best = sorted[0].stats;
+		const secondBest = sorted[1].stats;
+
+		// basically, it's only the best if the two means plus their moe are disjoint
+		if (best.mean + best.moe + secondBest.moe < secondBest.mean) {
+			return sorted[0];
+		}
+	}
+
+	return null;
+}
+/**
+ * @param {Result[]} results
+ * @returns {Result | null}
+ */
+function getWorst(results) {
+	if (results.length >= 2) {
+		const sorted = [...results].sort((a, b) => b.stats.mean - a.stats.mean);
+		const worst = sorted[0].stats;
+		const secondWorst = sorted[1].stats;
+
+		// basically, it's only the best if the two means plus their moe are disjoint
+		// (moe = margin of error; https://benchmarkjs.com/docs#stats_moe)
+		if (worst.mean - worst.moe - secondWorst.moe > secondWorst.mean) {
+			return sorted[0];
+		}
+	}
+
+	return null;
+}
+
+/**
+ * Create a new test function from the given Prism instance.
+ *
+ * @param {any} Prism
+ * @param {string} mainLanguage
+ * @param {string} testFunction
+ * @returns {(code: string) => void}
+ */
+function createTestFunction(Prism, mainLanguage, testFunction) {
+	if (testFunction === 'tokenize') {
+		return (code) => {
+			Prism.tokenize(code, Prism.languages[mainLanguage]);
+		};
+	} else if (testFunction === 'highlight') {
+		return (code) => {
+			Prism.highlight(code, Prism.languages[mainLanguage], mainLanguage);
+		};
+	} else {
+		throw new Error(`Unknown test function "${testFunction}"`);
+	}
+
+}
+
+/**
+ * @param {import("./config").Config} config
+ * @returns {Promise<Candidate[]>}
+ *
+ * @typedef Candidate
+ * @property {string} name
+ * @property {(mainLanguage: string, languages: string[]) => (code: string) => void} setup
+ */
+async function getCandidates(config) {
+	/** @type {Candidate[]} */
+	const candidates = [];
+
+	// local
+	if (!config.options.remotesOnly) {
+		const localPrismLoader = require('../tests/helper/prism-loader');
+		candidates.push({
+			name: 'local',
+			setup(mainLanguage, languages) {
+				const Prism = localPrismLoader.createInstance(languages);
+				return createTestFunction(Prism, mainLanguage, config.options.testFunction);
+			}
+		});
+	}
+
+	// remotes
+
+	// prepare base directory
+	const remoteBaseDir = path.join(__dirname, 'remotes');
+	await fs.promises.mkdir(remoteBaseDir, { recursive: true });
+
+	const baseGit = simpleGit(remoteBaseDir);
+
+	for (const remote of config.remotes) {
+		const user = /[^/]+(?=\/prism.git)/.exec(remote.repo);
+		const branch = remote.branch || 'master';
+		const remoteName = `${user}@${branch}`;
+		const remoteDir = path.join(remoteBaseDir, `${user}@${branch}`);
+
+		let remoteGit;
+		if (!fs.existsSync(remoteDir)) {
+			console.log(`Cloning ${remote.repo}`);
+			await baseGit.clone(remote.repo, remoteName);
+			remoteGit = simpleGit(remoteDir);
+		} else {
+			remoteGit = simpleGit(remoteDir);
+			await remoteGit.fetch('origin', branch); // get latest version of branch
+		}
+		await remoteGit.checkout(branch); // switch to branch
+
+		const remotePrismLoader = require(path.join(remoteDir, 'tests/helper/prism-loader'));
+		candidates.push({
+			name: remoteName,
+			setup(mainLanguage, languages) {
+				const Prism = remotePrismLoader.createInstance(languages);
+				return createTestFunction(Prism, mainLanguage, config.options.testFunction);
+			}
+		});
+	}
+
+	return candidates;
+}
+
+/**
+ * A utility function that converts the given optional array-like value into an array.
+ *
+ * @param {T[] | T | undefined | null} value
+ * @returns {readonly T[]}
+ * @template T
+ */
+function toArray(value) {
+	if (Array.isArray(value)) {
+		return value;
+	} else if (value != undefined) {
+		return [value];
+	} else {
+		return [];
+	}
+}
+
+
+runBenchmark(getConfig());
diff --git a/benchmark/config.js b/benchmark/config.js
new file mode 100644
index 0000000..1fcf386
--- /dev/null
+++ b/benchmark/config.js
@@ -0,0 +1,104 @@
+/**
+ * @type {Config}
+ *
+ * @typedef Config
+ * @property {ConfigOptions} options
+ * @property {ConfigRemote[]} remotes
+ * @property {Object<string, ConfigCase>} cases
+ *
+ * @typedef ConfigOptions
+ * @property {'tokenize' | 'highlight'} testFunction
+ * @property {number} maxTime in seconds
+ * @property {string} [language] An optional comma separated list of languages than, if defined, will be the only
+ * languages for which the benchmark will be run.
+ * @property {boolean} [remotesOnly=false] Whether the benchmark will only run with remotes. If `true`, the local
+ * project will be ignored
+ *
+ * @typedef ConfigRemote
+ * @property {string} repo
+ * @property {string} [branch='master']
+ *
+ * @typedef ConfigCase
+ * @property {string | string[]} [extends]
+ * @property {string | string[]} [files]
+ */
+const config = {
+	options: {
+		testFunction: 'tokenize',
+		maxTime: 3,
+		remotesOnly: false
+	},
+
+	remotes: [
+		/**
+		 * This will checkout a specific branch from a given repo.
+		 *
+		 * If no branch is specified, the master branch will be used.
+		 */
+
+		{
+			repo: 'https://github.com/PrismJS/prism.git'
+		},
+		/*{
+			repo: 'https://github.com/<Your user name>/prism.git',
+			branch: 'some-brach-you-want-to-test'
+		},*/
+	],
+
+	cases: {
+		'css': {
+			files: [
+				'../assets/style.css'
+			]
+		},
+		'css!+css-extras': { extends: 'css' },
+		'javascript': {
+			extends: 'json',
+			files: [
+				'https://cdnjs.cloudflare.com/ajax/libs/prism/1.20.0/prism.js',
+				'https://cdnjs.cloudflare.com/ajax/libs/prism/1.20.0/prism.min.js',
+				'https://code.jquery.com/jquery-3.4.1.js',
+				'https://code.jquery.com/jquery-3.4.1.min.js',
+				'../assets/vendor/utopia.js'
+			]
+		},
+		'json': {
+			files: [
+				'../components.json',
+				'../package-lock.json'
+			]
+		},
+		'markup': {
+			files: [
+				'../download.html',
+				'../index.html',
+				'https://github.com/PrismJS/prism', // the PrismJS/prism GitHub page
+			]
+		},
+		'markup!+css+javascript': { extends: 'markup' },
+		'c': {
+			files: [
+				'https://raw.githubusercontent.com/git/git/master/remote.h',
+				'https://raw.githubusercontent.com/git/git/master/remote.c',
+				'https://raw.githubusercontent.com/git/git/master/mergesort.c',
+				'https://raw.githubusercontent.com/git/git/master/mergesort.h'
+			]
+		},
+		'ruby': {
+			files: [
+				'https://raw.githubusercontent.com/rails/rails/master/actionview/lib/action_view/base.rb',
+				'https://raw.githubusercontent.com/rails/rails/master/actionview/lib/action_view/layouts.rb',
+				'https://raw.githubusercontent.com/rails/rails/master/actionview/lib/action_view/template.rb',
+			]
+		},
+		'rust': {
+			files: [
+				'https://raw.githubusercontent.com/rust-lang/regex/master/src/utf8.rs',
+				'https://raw.githubusercontent.com/rust-lang/regex/master/src/compile.rs',
+				'https://raw.githubusercontent.com/rust-lang/regex/master/src/lib.rs'
+			]
+		}
+	}
+};
+
+module.exports = config;
diff --git a/bower.json b/bower.json
index e95a321..86b55dd 100644
--- a/bower.json
+++ b/bower.json
@@ -17,6 +17,7 @@
     "assets",
     "docs",
     "tests",
+    "benchmark",
     "CNAME",
     "*.html",
     "*.svg"
diff --git a/package-lock.json b/package-lock.json
index 66c970b..afb6030 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -351,6 +351,38 @@
       "integrity": "sha512-5tabW/i+9mhrfEOUcLDu2xBPsHJ+X5Orqy9FKpale3SjDA17j5AEpYq5vfy3oAeAHGcvANRCO3NV3d2D6q3NiA==",
       "dev": true
     },
+    "@types/node-fetch": {
+      "version": "2.5.7",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.7.tgz",
+      "integrity": "sha512-o2WVNf5UhWRkxlf6eq+jMZDu7kjgpgJfl4xVNlvryc95O/6F2ld8ztKX+qu+Rjyet93WAWm5LjeX9H5FGkODvw==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      },
+      "dependencies": {
+        "combined-stream": {
+          "version": "1.0.8",
+          "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+          "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+          "dev": true,
+          "requires": {
+            "delayed-stream": "~1.0.0"
+          }
+        },
+        "form-data": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
+          "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
+          "dev": true,
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+          }
+        }
+      }
+    },
     "a-sync-waterfall": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz",
@@ -863,6 +895,16 @@
       "integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==",
       "dev": true
     },
+    "benchmark": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/benchmark/-/benchmark-2.1.4.tgz",
+      "integrity": "sha1-CfPeMckWQl1JjMLuVloOvzwqVik=",
+      "dev": true,
+      "requires": {
+        "lodash": "^4.17.4",
+        "platform": "^1.3.3"
+      }
+    },
     "binary-extensions": {
       "version": "1.13.1",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
@@ -5916,6 +5958,12 @@
       "integrity": "sha1-DPd1eml38b9/ajIge3CeN3OI6HQ=",
       "dev": true
     },
+    "platform": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz",
+      "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==",
+      "dev": true
+    },
     "pn": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz",
diff --git a/package.json b/package.json
index a598a93..afea480 100755
--- a/package.json
+++ b/package.json
@@ -5,6 +5,7 @@
   "main": "prism.js",
   "style": "themes/prism.css",
   "scripts": {
+    "benchmark": "node benchmark/benchmark.js",
     "build": "gulp",
     "start": "http-server -c-1",
     "lint": "eslint . --cache",
@@ -33,6 +34,8 @@
   "license": "MIT",
   "readmeFilename": "README.md",
   "devDependencies": {
+    "@types/node-fetch": "^2.5.5",
+    "benchmark": "^2.1.4",
     "chai": "^4.2.0",
     "danger": "^10.5.0",
     "del": "^4.1.1",
@@ -52,6 +55,7 @@
     "http-server": "^0.12.3",
     "jsdom": "^13.0.0",
     "mocha": "^6.2.0",
+    "node-fetch": "^2.6.0",
     "npm-run-all": "^4.1.5",
     "pump": "^3.0.0",
     "refa": "^0.9.1",
kmx.io/prism.js

Commit 44456b21d0bc6aeea84bffe7622b43ffb69f9591
