diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a8cdc86..0000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "src/test/resources/html5lib-tests"] - path = src/test/resources/html5lib-tests - url = https://github.com/html5lib/html5lib-tests.git -[submodule "src/test/resources/wpt"] - path = src/test/resources/wpt - url = https://github.com/web-platform-tests/wpt.git diff --git a/external-resources/commit.md b/external-resources/commit.md new file mode 100644 index 0000000..6f558d0 --- /dev/null +++ b/external-resources/commit.md @@ -0,0 +1,2 @@ +https://github.com/html5lib/html5lib-tests.git 224991ec10db04f056a89eed8b0bd8695fd2950e +https://github.com/web-platform-tests/wpt.git 73dff7c1179ee471bee7e005cdc8fc854ed76885 diff --git a/external-resources/html5lib-tests/tokenizer/README.md b/external-resources/html5lib-tests/tokenizer/README.md new file mode 100644 index 0000000..66b81e8 --- /dev/null +++ b/external-resources/html5lib-tests/tokenizer/README.md @@ -0,0 +1,107 @@ +Tokenizer tests +=============== + +The test format is [JSON](http://www.json.org/). This has the advantage +that the syntax allows backward-compatible extensions to the tests and +the disadvantage that it is relatively verbose. + +Basic Structure +--------------- + + {"tests": [ + {"description": "Test description", + "input": "input_string", + "output": [expected_output_tokens], + "initialStates": [initial_states], + "lastStartTag": last_start_tag, + "errors": [parse_errors] + } + ]} + +Multiple tests per file are allowed simply by adding more objects to the +"tests" list. + +Each parse error is an object that contains error `code` and one-based +error location indices: `line` and `col`. + +`description`, `input` and `output` are always present. The other values +are optional. + +### Test set-up + +`test.input` is a string containing the characters to pass to the +tokenizer. Specifically, it represents the characters of the **input +stream**, and so implementations are expected to perform the processing +described in the spec's **Preprocessing the input stream** section +before feeding the result to the tokenizer. + +If `test.doubleEscaped` is present and `true`, then `test.input` is not +quite as described above. Instead, it must first be subjected to another +round of unescaping (i.e., in addition to any unescaping involved in the +JSON import), and the result of *that* represents the characters of the +input stream. Currently, the only unescaping required by this option is +to convert each sequence of the form \\uHHHH (where H is a hex digit) +into the corresponding Unicode code point. (Note that this option also +affects the interpretation of `test.output`.) + +`test.initialStates` is a list of strings, each being the name of a +tokenizer state which can be one of the following: + +- `Data state` +- `PLAINTEXT state` +- `RCDATA state` +- `RAWTEXT state` +- `Script data state` +- `CDATA section state` + + The test should be run once for each string, using it +to set the tokenizer's initial state for that run. If +`test.initialStates` is omitted, it defaults to `["Data state"]`. + +`test.lastStartTag` is a lowercase string that should be used as "the +tag name of the last start tag to have been emitted from this +tokenizer", referenced in the spec's definition of **appropriate end tag +token**. If it is omitted, it is treated as if "no start tag has been +emitted from this tokenizer". + +### Test results + +`test.output` is a list of tokens, ordered with the first produced by +the tokenizer the first (leftmost) in the list. The list must mach the +**complete** list of tokens that the tokenizer should produce. Valid +tokens are: + + ["DOCTYPE", name, public_id, system_id, correctness] + ["StartTag", name, {attributes}*, true*] + ["StartTag", name, {attributes}] + ["EndTag", name] + ["Comment", data] + ["Character", data] + +`public_id` and `system_id` are either strings or `null`. `correctness` +is either `true` or `false`; `true` corresponds to the force-quirks flag +being false, and vice-versa. + +When the self-closing flag is set, the `StartTag` array has `true` as +its fourth entry. When the flag is not set, the array has only three +entries for backwards compatibility. + +All adjacent character tokens are coalesced into a single +`["Character", data]` token. + +If `test.doubleEscaped` is present and `true`, then every string within +`test.output` must be further unescaped (as described above) before +comparing with the tokenizer's output. + +xmlViolation tests +------------------ + +`tokenizer/xmlViolation.test` differs from the above in a couple of +ways: + +- The name of the single member of the top-level JSON object is + "xmlViolationTests" instead of "tests". +- Each test's expected output assumes that implementation is applying + the tweaks given in the spec's "Coercing an HTML DOM into an + infoset" section. + diff --git a/external-resources/html5lib-tests/tokenizer/contentModelFlags.test b/external-resources/html5lib-tests/tokenizer/contentModelFlags.test new file mode 100644 index 0000000..9cf7c8b --- /dev/null +++ b/external-resources/html5lib-tests/tokenizer/contentModelFlags.test @@ -0,0 +1,93 @@ +{"tests": [ + +{"description":"PLAINTEXT content model flag", +"initialStates":["PLAINTEXT state"], +"lastStartTag":"plaintext", +"input":"
&body;", +"output":[["Character", "&body;"]]}, + +{"description":"PLAINTEXT with seeming close tag", +"initialStates":["PLAINTEXT state"], +"lastStartTag":"plaintext", +"input":"&body;", +"output":[["Character", "&body;"]]}, + +{"description":"End tag closing RCDATA or RAWTEXT", +"initialStates":["RCDATA state", "RAWTEXT state"], +"lastStartTag":"xmp", +"input":"foo", +"output":[["Character", "foo"], ["EndTag", "xmp"]]}, + +{"description":"End tag closing RCDATA or RAWTEXT (case-insensitivity)", +"initialStates":["RCDATA state", "RAWTEXT state"], +"lastStartTag":"xmp", +"input":"foo", +"output":[["Character", "foo"], ["EndTag", "xmp"]]}, + +{"description":"End tag closing RCDATA or RAWTEXT (ending with space)", +"initialStates":["RCDATA state", "RAWTEXT state"], +"lastStartTag":"xmp", +"input":"foobar", +"output":[["Character", "bar"], ["EndTag", "xmp"]]}, + +{"description":"Partial end tags leading straight into partial end tags", +"initialStates":["RCDATA state", "RAWTEXT state"], +"lastStartTag":"xmp", +"input":"", +"output":[["Character", "bar", +"output":[["Character", "bar"]]}, + +{"description":"End tag closing RCDATA or RAWTEXT, switching back to PCDATA", +"initialStates":["RCDATA state", "RAWTEXT state"], +"lastStartTag":"xmp", +"input":"foo", +"output":[["Character", "foo"], ["EndTag", "xmp"], ["EndTag", "baz"]]}, + +{"description":"RAWTEXT w/ something looking like an entity", +"initialStates":["RAWTEXT state"], +"lastStartTag":"xmp", +"input":"&foo;", +"output":[["Character", "&foo;"]]}, + +{"description":"RCDATA w/ an entity", +"initialStates":["RCDATA state"], +"lastStartTag":"textarea", +"input":"<", +"output":[["Character", "<"]]} + +]} diff --git a/external-resources/html5lib-tests/tokenizer/domjs.test b/external-resources/html5lib-tests/tokenizer/domjs.test new file mode 100644 index 0000000..1a0824d --- /dev/null +++ b/external-resources/html5lib-tests/tokenizer/domjs.test @@ -0,0 +1,335 @@ +{ + "tests": [ + { + "description":"CR in bogus comment state", + "input":"\u000d", + "output":[["Comment", "?\u000a"]], + "errors":[ + { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 } + ] + }, + { + "description":"CRLF in bogus comment state", + "input":"\u000d\u000a", + "output":[["Comment", "?\u000a"]], + "errors":[ + { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 } + ] + }, + { + "description":"CRLFLF in bogus comment state", + "input":"\u000d\u000a\u000a", + "output":[["Comment", "?\u000a\u000a"]], + "errors":[ + { "code": "unexpected-question-mark-instead-of-tag-name", "line": 1, "col": 2 } + ] + }, + { + "description":"Raw NUL replacement", + "doubleEscaped":true, + "initialStates":["RCDATA state", "RAWTEXT state", "PLAINTEXT state", "Script data state"], + "input":"\\u0000", + "output":[["Character", "\\uFFFD"]], + "errors":[ + { "code": "unexpected-null-character", "line": 1, "col": 1 } + ] + }, + { + "description":"NUL in CDATA section", + "doubleEscaped":true, + "initialStates":["CDATA section state"], + "input":"\\u0000]]>", + "output":[["Character", "\\u0000"]] + }, + { + "description":"NUL in script HTML comment", + "doubleEscaped":true, + "initialStates":["Script data state"], + "input":"", + "output":[["Character", ""]], + "errors":[ + { "code": "unexpected-null-character", "line": 1, "col": 9 }, + { "code": "unexpected-null-character", "line": 1, "col": 22 }, + { "code": "unexpected-null-character", "line": 1, "col": 36 } + ] + }, + { + "description":"NUL in script HTML comment - double escaped", + "doubleEscaped":true, + "initialStates":["Script data state"], + "input":"", + "output":[["Character", ""]], + "errors":[ + { "code": "unexpected-null-character", "line": 1, "col": 13 }, + { "code": "unexpected-null-character", "line": 1, "col": 30 }, + { "code": "unexpected-null-character", "line": 1, "col": 48 } + ] + }, + { + "description":"EOF in script HTML comment", + "initialStates":["Script data state"], + "input":"", + "output":[["Character", ""]] + }, + { + "description":"Dash less-than in script HTML comment", + "initialStates":["Script data state"], + "input":"", + "output":[["Character", ""]] + }, + { + "description":"Dash at end of script HTML comment", + "initialStates":["Script data state"], + "input":"", + "output":[["Character", ""]] + }, + { + "description":" in script HTML comment", + "initialStates":["Script data state"], + "lastStartTag":"script", + "input":"", + "output":[["Character", ""], ["EndTag", "script"]] + }, + { + "description":" in script HTML comment - double escaped", + "initialStates":["Script data state"], + "lastStartTag":"script", + "input":"", + "output":[["Character", ""], ["EndTag", "script"]] + }, + { + "description":" in script HTML comment - double escaped with nested -->", + "output":[["Character", ""], ["EndTag", "script"]] + }, + { + "description":" in script HTML comment - double escaped with abrupt end", + "initialStates":["Script data state"], + "lastStartTag":"script", + "input":" -->", + "output":[["Character", ""], ["EndTag", "script"], ["Character", " -->"], ["EndTag", "script"]] + }, + { + "description":"Incomplete start tag in script HTML comment double escaped", + "initialStates":["Script data state"], + "lastStartTag":"script", + "input":"", + "output":[["Character", ""]] + }, + { + "description":"Unclosed start tag in script HTML comment double escaped", + "initialStates":["Script data state"], + "lastStartTag":"script", + "input":"", + "output":[["Character", ""]] + }, + { + "description":"Incomplete end tag in script HTML comment double escaped", + "initialStates":["Script data state"], + "lastStartTag":"script", + "input":"", + "output":[["Character", ""]] + }, + { + "description":"Unclosed end tag in script HTML comment double escaped", + "initialStates":["Script data state"], + "lastStartTag":"script", + "input":"", + "output":[["Character", ""]] + }, + { + "description":"leading U+FEFF must pass through", + "initialStates":["Data state", "RCDATA state", "RAWTEXT state", "Script data state"], + "doubleEscaped":true, + "input":"\\uFEFFfoo\\uFEFFbar", + "output":[["Character", "\\uFEFFfoo\\uFEFFbar"]] + }, + { + "description":"Non BMP-charref in RCDATA", + "initialStates":["RCDATA state"], + "input":"≂̸", + "output":[["Character", "\u2242\u0338"]] + }, + { + "description":"Bad charref in RCDATA", + "initialStates":["RCDATA state"], + "input":"&NotEqualTild;", + "output":[["Character", "&NotEqualTild;"]], + "errors":[ + { "code": "unknown-named-character-reference", "line": 1, "col": 14 } + ] + }, + { + "description":"lowercase endtags", + "initialStates":["RCDATA state", "RAWTEXT state", "Script data state"], + "lastStartTag":"xmp", + "input":"", + "output":[["EndTag","xmp"]] + }, + { + "description":"bad endtag (space before name)", + "initialStates":["RCDATA state", "RAWTEXT state", "Script data state"], + "lastStartTag":"xmp", + "input":" XMP>", + "output":[["Character"," XMP>"]] + }, + { + "description":"bad endtag (not matching last start tag)", + "initialStates":["RCDATA state", "RAWTEXT state", "Script data state"], + "lastStartTag":"xmp", + "input":"", + "output":[["Character",""]] + }, + { + "description":"bad endtag (without close bracket)", + "initialStates":["RCDATA state", "RAWTEXT state", "Script data state"], + "lastStartTag":"xmp", + "input":"", + "output":[["StartTag", "p", {"id":"\u2242\u0338"}]] + }, + { + "description":"--!NUL in comment ", + "doubleEscaped":true, + "input":"", + "output":[["Comment", "--!\\uFFFD"]], + "errors":[ + { "code": "unexpected-null-character", "line": 1, "col": 8 } + ] + }, + { + "description":"space EOF after doctype ", + "input":"", + "output":[["Comment", "[CDATA[foo]]"]], + "errors":[ + { "code": "cdata-in-html-content", "line": 1, "col": 9 } + ] + }, + { + "description":"CDATA content", + "input":"foo ]]>", + "initialStates":["CDATA section state"], + "output":[["Character", "foo "]] + }, + { + "description":"CDATA followed by HTML content", + "input":"foo ]]> ", + "initialStates":["CDATA section state"], + "output":[["Character", "foo "]] + }, + { + "description":"CDATA with extra bracket", + "input":"foo]]]>", + "initialStates":["CDATA section state"], + "output":[["Character", "foo]"]] + }, + { + "description":"CDATA without end marker", + "input":"foo", + "initialStates":["CDATA section state"], + "output":[["Character", "foo"]], + "errors":[ + { "code": "eof-in-cdata", "line": 1, "col": 4 } + ] + }, + { + "description":"CDATA with single bracket ending", + "input":"foo]", + "initialStates":["CDATA section state"], + "output":[["Character", "foo]"]], + "errors":[ + { "code": "eof-in-cdata", "line": 1, "col": 5 } + ] + }, + { + "description":"CDATA with two brackets ending", + "input":"foo]]", + "initialStates":["CDATA section state"], + "output":[["Character", "foo]]"]], + "errors":[ + { "code": "eof-in-cdata", "line": 1, "col": 6 } + ] + }, + { + "description": "HTML tag in script data", + "input": "hello world", + "initialStates": ["Script data state"], + "output": [["Character", "hello world"]] + } + ] +} diff --git a/external-resources/html5lib-tests/tokenizer/entities.test b/external-resources/html5lib-tests/tokenizer/entities.test new file mode 100644 index 0000000..a6469cd --- /dev/null +++ b/external-resources/html5lib-tests/tokenizer/entities.test @@ -0,0 +1,542 @@ +{"tests": [ + +{"description": "Undefined named entity in a double-quoted attribute value ending in semicolon and whose name starts with a known entity name.", +"input":"One
Two + #errors + 3: Missing document type declaration + #document + | + |
+ | + |+ | "One" + |
+ | "Two" diff --git a/external-resources/wpt/html/syntax/parsing/resources/adoption01.dat b/external-resources/wpt/html/syntax/parsing/resources/adoption01.dat new file mode 100644 index 0000000..38f98ef --- /dev/null +++ b/external-resources/wpt/html/syntax/parsing/resources/adoption01.dat @@ -0,0 +1,354 @@ +#data +
+#errors +(1,3): expected-doctype-but-got-start-tag +(1,10): adoption-agency-1.3 +#document +| +| +| +| +|
+|
+
+#data
+1 2
+| +| "2" +| "3" + +#data +13 +#errors +(1,3): expected-doctype-but-got-start-tag +(1,17): adoption-agency-1.3 +#document +| +|
+| +| +| "1" +|