Commit a943f2bbb9c81d16fc8386c3b7df456ccc458046

Michael Schmidt 2021-12-18T13:26:17

CSP: Improved tokenization (#3276)

diff --git a/components/prism-csp.js b/components/prism-csp.js
index c7229e7..8030886 100644
--- a/components/prism-csp.js
+++ b/components/prism-csp.js
@@ -4,26 +4,73 @@
  * Reference: https://scotthelme.co.uk/csp-cheat-sheet/
  *
  * Supports the following:
- *  - CSP Level 1
- *  - CSP Level 2
- *  - CSP Level 3
+ *  - https://www.w3.org/TR/CSP1/
+ *  - https://www.w3.org/TR/CSP2/
+ *  - https://www.w3.org/TR/CSP3/
  */
 
-Prism.languages.csp = {
-	'directive': {
-		pattern: /(^|[^-\da-z])(?:base-uri|block-all-mixed-content|(?:child|connect|default|font|frame|img|manifest|media|object|prefetch|script|style|worker)-src|disown-opener|form-action|frame-(?:ancestors|options)|input-protection(?:-(?:clip|selectors))?|navigate-to|plugin-types|policy-uri|referrer|reflected-xss|report-(?:to|uri)|require-sri-for|sandbox|(?:script|style)-src-(?:attr|elem)|upgrade-insecure-requests)(?=[^-\da-z]|$)/i,
-		lookbehind: true,
-		alias: 'keyword'
-	},
-	'safe': {
-		// CSP2 hashes and nonces are base64 values. CSP3 accepts both base64 and base64url values.
-		// See https://tools.ietf.org/html/rfc4648#section-4
-		// See https://tools.ietf.org/html/rfc4648#section-5
-		pattern: /'(?:deny|none|report-sample|self|strict-dynamic|top-only|(?:nonce|sha(?:256|384|512))-[-+/\w=]+)'/i,
-		alias: 'selector'
-	},
-	'unsafe': {
-		pattern: /(?:'unsafe-(?:allow-redirects|dynamic|eval|hash-attributes|hashed-attributes|hashes|inline)'|\*)/i,
-		alias: 'function'
+(function (Prism) {
+
+	/**
+	 * @param {string} source
+	 * @returns {RegExp}
+	 */
+	function value(source) {
+		return RegExp(/([ \t])/.source + '(?:' + source + ')' + /(?=[\s;]|$)/.source, 'i');
 	}
-};
+
+	Prism.languages.csp = {
+		'directive': {
+			pattern: /(^|[\s;])(?:base-uri|block-all-mixed-content|(?:child|connect|default|font|frame|img|manifest|media|object|prefetch|script|style|worker)-src|disown-opener|form-action|frame-(?:ancestors|options)|input-protection(?:-(?:clip|selectors))?|navigate-to|plugin-types|policy-uri|referrer|reflected-xss|report-(?:to|uri)|require-sri-for|sandbox|(?:script|style)-src-(?:attr|elem)|upgrade-insecure-requests)(?=[\s;]|$)/i,
+			lookbehind: true,
+			alias: 'property'
+		},
+		'scheme': {
+			pattern: value(/[a-z][a-z0-9.+-]*:/.source),
+			lookbehind: true
+		},
+		'none': {
+			pattern: value(/'none'/.source),
+			lookbehind: true,
+			alias: 'keyword'
+		},
+		'nonce': {
+			pattern: value(/'nonce-[-+/\w=]+'/.source),
+			lookbehind: true,
+			alias: 'number'
+		},
+		'hash': {
+			pattern: value(/'sha(?:256|384|512)-[-+/\w=]+'/.source),
+			lookbehind: true,
+			alias: 'number'
+		},
+		'host': {
+			pattern: value(
+				/[a-z][a-z0-9.+-]*:\/\/[^\s;,']*/.source +
+				'|' +
+				/\*[^\s;,']*/.source +
+				'|' +
+				/[a-z0-9-]+(?:\.[a-z0-9-]+)+(?::[\d*]+)?(?:\/[^\s;,']*)?/.source
+			),
+			lookbehind: true,
+			alias: 'url',
+			inside: {
+				'important': /\*/
+			}
+		},
+		'keyword': [
+			{
+				pattern: value(/'unsafe-[a-z-]+'/.source),
+				lookbehind: true,
+				alias: 'unsafe'
+			},
+			{
+				pattern: value(/'[a-z-]+'/.source),
+				lookbehind: true,
+				alias: 'safe'
+			},
+		],
+		'punctuation': /;/
+	};
+
+}(Prism));
diff --git a/components/prism-csp.min.js b/components/prism-csp.min.js
index 6dc0715..c415e34 100644
--- a/components/prism-csp.min.js
+++ b/components/prism-csp.min.js
@@ -1 +1 @@
-Prism.languages.csp={directive:{pattern:/(^|[^-\da-z])(?:base-uri|block-all-mixed-content|(?:child|connect|default|font|frame|img|manifest|media|object|prefetch|script|style|worker)-src|disown-opener|form-action|frame-(?:ancestors|options)|input-protection(?:-(?:clip|selectors))?|navigate-to|plugin-types|policy-uri|referrer|reflected-xss|report-(?:to|uri)|require-sri-for|sandbox|(?:script|style)-src-(?:attr|elem)|upgrade-insecure-requests)(?=[^-\da-z]|$)/i,lookbehind:!0,alias:"keyword"},safe:{pattern:/'(?:deny|none|report-sample|self|strict-dynamic|top-only|(?:nonce|sha(?:256|384|512))-[-+/\w=]+)'/i,alias:"selector"},unsafe:{pattern:/(?:'unsafe-(?:allow-redirects|dynamic|eval|hash-attributes|hashed-attributes|hashes|inline)'|\*)/i,alias:"function"}};
\ No newline at end of file
+!function(e){function n(e){return RegExp("([ \t])(?:"+e+")(?=[\\s;]|$)","i")}Prism.languages.csp={directive:{pattern:/(^|[\s;])(?:base-uri|block-all-mixed-content|(?:child|connect|default|font|frame|img|manifest|media|object|prefetch|script|style|worker)-src|disown-opener|form-action|frame-(?:ancestors|options)|input-protection(?:-(?:clip|selectors))?|navigate-to|plugin-types|policy-uri|referrer|reflected-xss|report-(?:to|uri)|require-sri-for|sandbox|(?:script|style)-src-(?:attr|elem)|upgrade-insecure-requests)(?=[\s;]|$)/i,lookbehind:!0,alias:"property"},scheme:{pattern:n("[a-z][a-z0-9.+-]*:"),lookbehind:!0},none:{pattern:n("'none'"),lookbehind:!0,alias:"keyword"},nonce:{pattern:n("'nonce-[-+/\\w=]+'"),lookbehind:!0,alias:"number"},hash:{pattern:n("'sha(?:256|384|512)-[-+/\\w=]+'"),lookbehind:!0,alias:"number"},host:{pattern:n("[a-z][a-z0-9.+-]*://[^\\s;,']*|\\*[^\\s;,']*|[a-z0-9-]+(?:\\.[a-z0-9-]+)+(?::[\\d*]+)?(?:/[^\\s;,']*)?"),lookbehind:!0,alias:"url",inside:{important:/\*/}},keyword:[{pattern:n("'unsafe-[a-z-]+'"),lookbehind:!0,alias:"unsafe"},{pattern:n("'[a-z-]+'"),lookbehind:!0,alias:"safe"}],punctuation:/;/}}();
\ No newline at end of file
diff --git a/tests/languages/csp/directive_no_value_feature.test b/tests/languages/csp/directive_no_value_feature.test
index a45d608..a9fce90 100644
--- a/tests/languages/csp/directive_no_value_feature.test
+++ b/tests/languages/csp/directive_no_value_feature.test
@@ -4,7 +4,7 @@ upgrade-insecure-requests;
 
 [
 	["directive", "upgrade-insecure-requests"],
-	";"
+	["punctuation", ";"]
 ]
 
 ----------------------------------------------------
diff --git a/tests/languages/csp/directive_with_source_expression_feature.test b/tests/languages/csp/directive_with_source_expression_feature.test
index f618d29..747dcc6 100644
--- a/tests/languages/csp/directive_with_source_expression_feature.test
+++ b/tests/languages/csp/directive_with_source_expression_feature.test
@@ -4,21 +4,26 @@ input-protection tolerance=50; input-protection-clip before=60; input-protection
 
 [
 	["directive", "input-protection"],
-	" tolerance=50; ",
+	" tolerance=50",
+	["punctuation", ";"],
 	["directive", "input-protection-clip"],
-	" before=60; ",
+	" before=60",
+	["punctuation", ";"],
 	["directive", "input-protection-selectors"],
-	" div; ",
+	" div",
+	["punctuation", ";"],
 	["directive", "policy-uri"],
-	" https://example.com; ",
+	["host", ["https://example.com"]],
+	["punctuation", ";"],
 	["directive", "script-src"],
-	" example.com; ",
+	["host", ["example.com"]],
+	["punctuation", ";"],
 	["directive", "script-src-attr"],
-	["safe", "'none'"],
-	"; ",
+	["none", "'none'"],
+	["punctuation", ";"],
 	["directive", "style-src-elem"],
-	["safe", "'none'"],
-	";"
+	["none", "'none'"],
+	["punctuation", ";"]
 ]
 
 ----------------------------------------------------
diff --git a/tests/languages/csp/hash_feature.test b/tests/languages/csp/hash_feature.test
new file mode 100644
index 0000000..fd3b130
--- /dev/null
+++ b/tests/languages/csp/hash_feature.test
@@ -0,0 +1,8 @@
+style-src 'sha256-EpOpN/ahUF6jhWShDUdy+NvvtaGcu5F7qM6+x2mfkh4='
+
+----------------------------------------------------
+
+[
+	["directive", "style-src"],
+	["hash", "'sha256-EpOpN/ahUF6jhWShDUdy+NvvtaGcu5F7qM6+x2mfkh4='"]
+]
diff --git a/tests/languages/csp/host_feature.test b/tests/languages/csp/host_feature.test
new file mode 100644
index 0000000..9bc19e5
--- /dev/null
+++ b/tests/languages/csp/host_feature.test
@@ -0,0 +1,46 @@
+default-src trusted.com *.trusted.com;
+img-src *;
+media-src media1.com media2.com;
+script-src userscripts.example.com;
+frame-ancestors https://alice https://bob;
+frame-ancestors https://example.com/;
+
+sandbox allow-scripts;
+
+----------------------------------------------------
+
+[
+	["directive", "default-src"],
+	["host", ["trusted.com"]],
+	["host", [
+		["important", "*"],
+		".trusted.com"
+	]],
+	["punctuation", ";"],
+
+	["directive", "img-src"],
+	["host", [
+		["important", "*"]
+	]],
+	["punctuation", ";"],
+
+	["directive", "media-src"],
+	["host", ["media1.com"]],
+	["host", ["media2.com"]],
+	["punctuation", ";"],
+
+	["directive", "script-src"],
+	["host", ["userscripts.example.com"]],
+	["punctuation", ";"],
+
+	["directive", "frame-ancestors"],
+	["host", ["https://alice"]],
+	["host", ["https://bob"]],
+	["punctuation", ";"],
+
+	["directive", "frame-ancestors"],
+	["host", ["https://example.com/"]],
+	["punctuation", ";"],
+
+	["directive", "sandbox"], " allow-scripts", ["punctuation", ";"]
+]
diff --git a/tests/languages/csp/issue2661.test b/tests/languages/csp/issue2661.test
index 1d25bd0..6eb7c5f 100644
--- a/tests/languages/csp/issue2661.test
+++ b/tests/languages/csp/issue2661.test
@@ -3,7 +3,10 @@ default-src-is-a-fake; fake-default-src;
 ----------------------------------------------------
 
 [
-	"default-src-is-a-fake; fake-default-src;"
+	"default-src-is-a-fake",
+	["punctuation", ";"],
+	" fake-default-src",
+	["punctuation", ";"]
 ]
 
 ----------------------------------------------------
diff --git a/tests/languages/csp/keyword_safe_feature.html.test b/tests/languages/csp/keyword_safe_feature.html.test
new file mode 100644
index 0000000..db3f36f
--- /dev/null
+++ b/tests/languages/csp/keyword_safe_feature.html.test
@@ -0,0 +1,16 @@
+default-src 'report-sample';
+style-src 'self' 'strict-dynamic';
+
+----------------------------------------------------
+
+<span class="token directive property">default-src</span>
+<span class="token keyword safe">'report-sample'</span>
+<span class="token punctuation">;</span>
+<span class="token directive property">style-src</span>
+<span class="token keyword safe">'self'</span>
+<span class="token keyword safe">'strict-dynamic'</span>
+<span class="token punctuation">;</span>
+
+----------------------------------------------------
+
+Checks for source expressions classified as safe.
diff --git a/tests/languages/csp/keyword_unsafe_feature.html.test b/tests/languages/csp/keyword_unsafe_feature.html.test
new file mode 100644
index 0000000..227bec6
--- /dev/null
+++ b/tests/languages/csp/keyword_unsafe_feature.html.test
@@ -0,0 +1,20 @@
+navigate-to 'unsafe-allow-redirects';
+script-src 'unsafe-dynamic' 'unsafe-eval' 'unsafe-hash-attributes' 'unsafe-hashed-attributes' 'unsafe-hashes' 'unsafe-inline';
+
+----------------------------------------------------
+
+<span class="token directive property">navigate-to</span>
+<span class="token keyword unsafe">'unsafe-allow-redirects'</span>
+<span class="token punctuation">;</span>
+<span class="token directive property">script-src</span>
+<span class="token keyword unsafe">'unsafe-dynamic'</span>
+<span class="token keyword unsafe">'unsafe-eval'</span>
+<span class="token keyword unsafe">'unsafe-hash-attributes'</span>
+<span class="token keyword unsafe">'unsafe-hashed-attributes'</span>
+<span class="token keyword unsafe">'unsafe-hashes'</span>
+<span class="token keyword unsafe">'unsafe-inline'</span>
+<span class="token punctuation">;</span>
+
+----------------------------------------------------
+
+Checks for source expressions classified as unsafe.
diff --git a/tests/languages/csp/nonce_feature.test b/tests/languages/csp/nonce_feature.test
new file mode 100644
index 0000000..dc0e4a3
--- /dev/null
+++ b/tests/languages/csp/nonce_feature.test
@@ -0,0 +1,9 @@
+style-src 'nonce-yeah';
+
+----------------------------------------------------
+
+[
+	["directive", "style-src"],
+	["nonce", "'nonce-yeah'"],
+	["punctuation", ";"]
+]
diff --git a/tests/languages/csp/none_feature.test b/tests/languages/csp/none_feature.test
new file mode 100644
index 0000000..7b4e837
--- /dev/null
+++ b/tests/languages/csp/none_feature.test
@@ -0,0 +1,8 @@
+sandbox 'none'
+
+----------------------------------------------------
+
+[
+	["directive", "sandbox"],
+	["none", "'none'"]
+]
diff --git a/tests/languages/csp/safe_feature.test b/tests/languages/csp/safe_feature.test
deleted file mode 100644
index f61cc32..0000000
--- a/tests/languages/csp/safe_feature.test
+++ /dev/null
@@ -1,20 +0,0 @@
-default-src 'none' 'report-sample'; style-src 'self' 'strict-dynamic' 'nonce-yeah' 'sha256-EpOpN/ahUF6jhWShDUdy+NvvtaGcu5F7qM6+x2mfkh4=';
-
-----------------------------------------------------
-
-[
-	["directive", "default-src"],
-	["safe", "'none'"],
-	["safe", "'report-sample'"],
-	"; ",
-	["directive", "style-src"],
-	["safe", "'self'"],
-	["safe", "'strict-dynamic'"],
-	["safe", "'nonce-yeah'"],
-	["safe", "'sha256-EpOpN/ahUF6jhWShDUdy+NvvtaGcu5F7qM6+x2mfkh4='"],
-	";"
-]
-
-----------------------------------------------------
-
-Checks for source expressions classified as safe.
diff --git a/tests/languages/csp/scheme_feature.test b/tests/languages/csp/scheme_feature.test
new file mode 100644
index 0000000..5807a8c
--- /dev/null
+++ b/tests/languages/csp/scheme_feature.test
@@ -0,0 +1,10 @@
+default-src https: 'unsafe-inline' 'unsafe-eval'
+
+----------------------------------------------------
+
+[
+	["directive", "default-src"],
+	["scheme", "https:"],
+	["keyword", "'unsafe-inline'"],
+	["keyword", "'unsafe-eval'"]
+]
diff --git a/tests/languages/csp/unsafe_feature.test b/tests/languages/csp/unsafe_feature.test
deleted file mode 100644
index 758ab58..0000000
--- a/tests/languages/csp/unsafe_feature.test
+++ /dev/null
@@ -1,21 +0,0 @@
-navigate-to 'unsafe-allow-redirects'; script-src 'unsafe-dynamic' 'unsafe-eval' 'unsafe-hash-attributes' 'unsafe-hashed-attributes' 'unsafe-hashes' 'unsafe-inline';
-
-----------------------------------------------------
-
-[
-	["directive", "navigate-to"],
-	["unsafe", "'unsafe-allow-redirects'"],
-	"; ",
-	["directive", "script-src"],
-	["unsafe", "'unsafe-dynamic'"],
-	["unsafe", "'unsafe-eval'"],
-	["unsafe", "'unsafe-hash-attributes'"],
-	["unsafe", "'unsafe-hashed-attributes'"],
-	["unsafe", "'unsafe-hashes'"],
-	["unsafe", "'unsafe-inline'"],
-	";"
-]
-
-----------------------------------------------------
-
-Checks for source expressions classified as unsafe.