From c8d86079343fc4d59c714b3bd324a3b42f9310df Mon Sep 17 00:00:00 2001 From: Louis Pilfold Date: Sun, 12 Apr 2026 23:59:29 +0100 Subject: [PATCH 1/7] Tokenisation --- gleam.toml | 1 + manifest.toml | 2 + src/tom.gleam | 1711 +++------------------------------- test/tom/tokenise_test.gleam | 85 ++ test/tom_test.gleam | 1063 +-------------------- 5 files changed, 207 insertions(+), 2655 deletions(-) create mode 100644 test/tom/tokenise_test.gleam diff --git a/gleam.toml b/gleam.toml index fa511d6..5d75a2e 100644 --- a/gleam.toml +++ b/gleam.toml @@ -12,6 +12,7 @@ links = [ [dependencies] gleam_stdlib = ">= 0.33.0 and < 3.0.0" gleam_time = ">= 1.2.0 and < 2.0.0" +splitter = ">= 1.2.0 and < 2.0.0" [dev-dependencies] gleeunit = "~> 1.0" diff --git a/manifest.toml b/manifest.toml index 0bedd3e..ebe1143 100644 --- a/manifest.toml +++ b/manifest.toml @@ -5,9 +5,11 @@ packages = [ { name = "gleam_stdlib", version = "0.68.1", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "F7FAEBD8EF260664E86A46C8DBA23508D1D11BB3BCC6EE1B89B3BC3E5C83FF1E" }, { name = "gleam_time", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_time", source = "hex", outer_checksum = "0DF3834D20193F0A38D0EB21F0A78D48F2EC276C285969131B86DF8D4EF9E762" }, { name = "gleeunit", version = "1.9.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleeunit", source = "hex", outer_checksum = "DA9553CE58B67924B3C631F96FE3370C49EB6D6DC6B384EC4862CC4AAA718F3C" }, + { name = "splitter", version = "1.2.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "splitter", source = "hex", outer_checksum = "3DFD6B6C49E61EDAF6F7B27A42054A17CFF6CA2135FF553D0CB61C234D281DD0" }, ] [requirements] gleam_stdlib = { version = ">= 0.33.0 and < 3.0.0" } gleam_time = { version = ">= 1.2.0 and < 2.0.0" } gleeunit = { version = "~> 1.0" } +splitter = { version = ">= 1.2.0 and < 2.0.0" } diff --git a/src/tom.gleam b/src/tom.gleam index d9b61be..875b7f5 100644 --- a/src/tom.gleam +++ b/src/tom.gleam @@ -1,63 +1,63 @@ -//// A pure Gleam TOML parser! -//// -//// ```gleam -//// import tom -//// -//// const config = " -//// [person] -//// name = \"Lucy\" -//// is_cool = true -//// " -//// -//// pub fn main() { -//// // Parse a string of TOML -//// let assert Ok(parsed) = tom.parse(config) -//// -//// // Now you can work with the data directly, or you can use the `get_*` -//// // functions to retrieve values. -//// -//// tom.get_string(parsed, ["person", "name"]) -//// // -> Ok("Lucy") -//// -//// let is_cool = tom.get_bool(parsed, ["person", "is_cool"]) -//// // -> Ok(True) -//// } -//// ``` - -import gleam/dict.{type Dict} -import gleam/float -import gleam/int import gleam/list +import gleam/option.{type Option, None, Some} import gleam/result import gleam/string -import gleam/time/calendar -import gleam/time/duration -import gleam/time/timestamp - -/// A TOML document. -pub type Toml { - Int(Int) - Float(Float) - /// Infinity is a valid number in TOML but Gleam does not support it, so this - /// variant represents the infinity values. - Infinity(Sign) - /// NaN is a valid number in TOML but Gleam does not support it, so this - /// variant represents the NaN values. - Nan(Sign) - Bool(Bool) - String(String) - Date(calendar.Date) - Time(calendar.TimeOfDay) - DateTime(date: calendar.Date, time: calendar.TimeOfDay, offset: Offset) - Array(List(Toml)) - ArrayOfTables(List(Dict(String, Toml))) - Table(Dict(String, Toml)) - InlineTable(Dict(String, Toml)) -} - -pub type Offset { - Local - Offset(duration.Duration) +import splitter.{type Splitter} + +/// A token produced by lexing TOML source text. +pub type Token { + WhitespaceToken(String) + NewlineToken + /// A comment starting with `#` and ending with `\n`. + CommentToken(String) + /// `=` + EqualsToken + /// `.` + DotToken + /// `,` + CommaToken + /// `{` + LeftBraceToken + /// `}` + RightBraceToken + /// `[` + LeftBracketToken + /// `]` + RightBracketToken + /// `[[` + DoubleLeftBracketToken + /// `]]` + DoubleRightBracketToken + /// A double-quote single-line string. + BasicStringToken(String) + /// A double-quote multi-line string. + MultiLineBasicStringToken(String) + /// A single-quote single-line string. + LiteralStringToken(String) + /// A single-quote multi-line string. + MultiLineLiteralStringToken(String) + /// An unquoted key segment e.g. `wibble` + BareKeyToken(String) + /// An int e.g `123` + IntToken(src: String, value: Int) + /// A float literal e.g. `123.456` + FloatToken(src: String, value: Float) + /// inf, -inf, +inf + InfinityToken(sign: Option(Sign)) + /// nan, -nan, +nan + NanToken(sign: Option(Sign)) + /// `true` or `false`. + BoolToken(value: Bool) + /// A date-time with an offset. + OffsetDateTimeToken(String) + /// A date-time with no offset. + LocalDateTimeToken(String) + /// A date. + LocalDateToken(String) + /// A time. + LocalTimeToken(String) + /// The end! + EndOfFile } pub type Sign { @@ -66,1570 +66,95 @@ pub type Sign { } /// An error that can occur when parsing a TOML document. -pub type ParseError { +pub type TomlError { /// An unexpected character was encountered when parsing the document. - Unexpected(got: String, expected: String) - /// More than one items have the same key in the document. - KeyAlreadyInUse(key: List(String)) -} - -type Tokens = - List(String) - -type Parsed(a) = - Result(#(a, Tokens), ParseError) - -/// A number of any kind, returned by the `get_number` function. -pub type Number { - NumberInt(Int) - NumberFloat(Float) - NumberInfinity(Sign) - NumberNan(Sign) -} - -/// An error that can occur when retrieving a value from a TOML document dictionary with -/// one of the `get_*` functions. -pub type GetError { - /// There was no value at the given key. - NotFound(key: List(String)) - /// The value at the given key was not of the expected type. - WrongType(key: List(String), expected: String, got: String) -} - -// TODO: test -/// Get a value of any type from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = 1") -/// get(parsed, ["a", "b", "c"]) -/// // -> Ok(Int(1)) -/// ``` -/// -pub fn get( - toml: Dict(String, Toml), - key: List(String), -) -> Result(Toml, GetError) { - case key { - [] -> Error(NotFound([])) - [k] -> result.replace_error(dict.get(toml, k), NotFound([k])) - [k, ..key] -> { - case dict.get(toml, k) { - Ok(Table(t)) -> push_key(get(t, key), k) - Ok(InlineTable(t)) -> push_key(get(t, key), k) - Ok(other) -> Error(WrongType([k], "Table", classify(other))) - Error(_) -> Error(NotFound([k])) - } - } - } -} - -// TODO: test -/// Get an int from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = 1") -/// get_int(parsed, ["a", "b", "c"]) -/// // -> Ok(1) -/// ``` -/// -pub fn get_int( - toml: Dict(String, Toml), - key: List(String), -) -> Result(Int, GetError) { - case get(toml, key) { - Ok(Int(i)) -> Ok(i) - Ok(other) -> Error(WrongType(key, "Int", classify(other))) - Error(e) -> Error(e) - } -} - -// TODO: test -/// Get a float from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = 1.1") -/// get_float(parsed, ["a", "b", "c"]) -/// // -> Ok(1.1) -/// ``` -/// -pub fn get_float( - toml: Dict(String, Toml), - key: List(String), -) -> Result(Float, GetError) { - case get(toml, key) { - Ok(Float(i)) -> Ok(i) - Ok(other) -> Error(WrongType(key, "Float", classify(other))) - Error(e) -> Error(e) - } -} - -// TODO: test -/// Get a bool from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = true") -/// get_bool(parsed, ["a", "b", "c"]) -/// // -> Ok(True) -/// ``` -/// -pub fn get_bool( - toml: Dict(String, Toml), - key: List(String), -) -> Result(Bool, GetError) { - case get(toml, key) { - Ok(Bool(i)) -> Ok(i) - Ok(other) -> Error(WrongType(key, "Bool", classify(other))) - Error(e) -> Error(e) - } -} - -// TODO: test -/// Get a string from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = \"ok\"") -/// get_string(parsed, ["a", "b", "c"]) -/// // -> Ok("ok") -/// ``` -/// -pub fn get_string( - toml: Dict(String, Toml), - key: List(String), -) -> Result(String, GetError) { - case get(toml, key) { - Ok(String(i)) -> Ok(i) - Ok(other) -> Error(WrongType(key, "String", classify(other))) - Error(e) -> Error(e) - } -} - -/// Get a date from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = 1979-05-27") -/// get_date(parsed, ["a", "b", "c"]) -/// // -> Ok(Date(1979, May, 27)) -/// ``` -/// -pub fn get_date( - toml: Dict(String, Toml), - key: List(String), -) -> Result(calendar.Date, GetError) { - case get(toml, key) { - Ok(Date(i)) -> Ok(i) - Ok(other) -> Error(WrongType(key, "Date", classify(other))) - Error(e) -> Error(e) - } -} - -/// Get a time from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = 07:32:00") -/// get_time_of_day(parsed, ["a", "b", "c"]) -/// // -> Ok(TimeOfDay(7, 32, 0, 0)) -/// ``` -/// -pub fn get_time_of_day( - toml: Dict(String, Toml), - key: List(String), -) -> Result(calendar.TimeOfDay, GetError) { - case get(toml, key) { - Ok(Time(i)) -> Ok(i) - Ok(other) -> Error(WrongType(key, "Time", classify(other))) - Error(e) -> Error(e) - } -} - -/// Get a date-time from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = 1979-05-27T07:32:00") -/// get_calendar_time(parsed, ["a", "b", "c"]) -/// // -> Ok(#(Date(1979, May, 27), TimeOfDay(7, 32, 0, 0), Local)) -/// ``` -/// -pub fn get_calendar_time( - toml: Dict(String, Toml), - key: List(String), -) -> Result(#(calendar.Date, calendar.TimeOfDay, Offset), GetError) { - case get(toml, key) { - Ok(DateTime(d, t, o)) -> Ok(#(d, t, o)) - Ok(other) -> Error(WrongType(key, "DateTime", classify(other))) - Error(e) -> Error(e) - } + Unexpected(byte_position: Int, got: String, expected: String) + // /// More than one items have the same key in the document. + // KeyAlreadyInUse(key: List(String)) } -/// Get an unambiguous time from a TOML document dictionary. -/// -/// If a TOML date time has no offset it is ambiguous and cannot be converted -/// into a timestamp. There's no way to know what actual point in time it would -/// be as it would be different in different time zones. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = 1970-00-00T00:00:00Z") -/// get_timestamp(parsed, ["a", "b", "c"]) -/// // -> Ok(timestamp.from_unix_seconds(0)) -/// ``` -/// -pub fn get_timestamp( - toml: Dict(String, Toml), - key: List(String), -) -> Result(timestamp.Timestamp, GetError) { - case get(toml, key) { - Ok(DateTime(date:, time:, offset: Offset(offset))) -> - Ok(timestamp.from_calendar(date, time, offset)) - Ok(other) -> Error(WrongType(key, "DateTime with offset", classify(other))) - Error(e) -> Error(e) - } +// TODO: document +pub fn to_tokens(src: String) -> Result(List(Token), TomlError) { + new_lexer(src) + |> fold_tokens([], fn(tokens, _, token) { Ok([token, ..tokens]) }) + |> result.map(list.reverse) } -// TODO: test -/// Get an array from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = [1, 2]") -/// get_array(parsed, ["a", "b", "c"]) -/// // -> Ok([Int(1), Int(2)]) -/// ``` -/// -pub fn get_array( - toml: Dict(String, Toml), - key: List(String), -) -> Result(List(Toml), GetError) { - case get(toml, key) { - Ok(Array(i)) -> Ok(i) - Ok(ArrayOfTables(i)) -> Ok(list.map(i, Table)) - Ok(other) -> Error(WrongType(key, "Array", classify(other))) - Error(e) -> Error(e) - } +fn new_lexer(src: String) -> Lexer { + let src = string.replace(src, "\r\n", "\n") + let splitters = Splitters(next: splitter.new(["\n", " ", "="])) + Lexer(0, src:, splitters:) } -// TODO: test -/// Get a table from a TOML document dictionary. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = { d = 1 }") -/// get_table(parsed, ["a", "b", "c"]) -/// // -> Ok(dict.from_list([#("d", Int(1))])) -/// ``` -/// -pub fn get_table( - toml: Dict(String, Toml), - key: List(String), -) -> Result(Dict(String, Toml), GetError) { - case get(toml, key) { - Ok(Table(i)) -> Ok(i) - Ok(InlineTable(i)) -> Ok(i) - Ok(other) -> Error(WrongType(key, "Table", classify(other))) - Error(e) -> Error(e) - } +fn advance(lexer: Lexer, src: String) -> Lexer { + let position = + lexer.position + string.byte_size(lexer.src) - string.byte_size(src) + Lexer(..lexer, position:, src:) } -// TODO: test -/// Get a number of any kind from a TOML document dictionary. -/// This could be an int, a float, a NaN, or an infinity. -/// -/// ## Examples -/// -/// ```gleam -/// let assert Ok(parsed) = parse("a.b.c = { d = inf }") -/// get_number(parsed, ["a", "b", "c"]) -/// // -> Ok(NumberInfinity(Positive))) -/// ``` -/// -pub fn get_number( - toml: Dict(String, Toml), - key: List(String), -) -> Result(Number, GetError) { - case get(toml, key) { - Ok(Int(x)) -> Ok(NumberInt(x)) - Ok(Float(x)) -> Ok(NumberFloat(x)) - Ok(Nan(x)) -> Ok(NumberNan(x)) - Ok(Infinity(x)) -> Ok(NumberInfinity(x)) - Ok(other) -> Error(WrongType(key, "Number", classify(other))) - Error(e) -> Error(e) - } +type Lexer { + Lexer(position: Int, src: String, splitters: Splitters) } -fn classify(toml: Toml) -> String { - case toml { - Int(_) -> "Int" - Float(_) -> "Float" - Nan(Positive) -> "NaN" - Nan(Negative) -> "Negative NaN" - Infinity(Positive) -> "Infinity" - Infinity(Negative) -> "Negative Infinity" - Bool(_) -> "Bool" - String(_) -> "String" - Date(_) -> "Date" - Time(_) -> "Time" - DateTime(_, _, _) -> "DateTime" - Array(_) -> "Array" - ArrayOfTables(_) -> "Array" - Table(_) -> "Table" - InlineTable(_) -> "Table" - } +type Splitters { + Splitters(next: Splitter) } -fn push_key(result: Result(t, GetError), key: String) -> Result(t, GetError) { - case result { - Ok(t) -> Ok(t) - Error(NotFound(path)) -> Error(NotFound([key, ..path])) - Error(WrongType(path, expected, got)) -> - Error(WrongType([key, ..path], expected, got)) - } -} - -pub fn parse(input: String) -> Result(Dict(String, Toml), ParseError) { - let input = string.to_graphemes(input) - let input = drop_comments(input, [], NotInString) - let input = skip_whitespace(input) - use toml, input <- do(parse_table(input, dict.new())) - case parse_tables(input, toml) { - Ok(toml) -> Ok(reverse_arrays_of_tables_table(toml)) - Error(e) -> Error(e) - } -} - -fn parse_tables( - input: Tokens, - toml: Dict(String, Toml), -) -> Result(Dict(String, Toml), ParseError) { - case input { - ["[", "[", ..input] -> { - case parse_array_of_tables(input) { - Error(e) -> Error(e) - Ok(#(#(key, table), input)) -> { - case insert(toml, key, ArrayOfTables([table])) { - Ok(toml) -> parse_tables(input, toml) - Error(e) -> Error(e) - } - } - } +fn fold_tokens( + lexer: Lexer, + output: output, + reduce: fn(output, Int, Token) -> Result(output, TomlError), +) -> Result(output, TomlError) { + case next_token(lexer) { + Ok(#(lexer, EndOfFile as token)) -> { + reduce(output, lexer.position, token) } - ["[", ..input] -> { - case parse_table_and_header(input) { - Error(e) -> Error(e) - Ok(#(#(key, table), input)) -> { - case insert(toml, key, Table(table)) { - Ok(toml) -> parse_tables(input, toml) - Error(e) -> Error(e) - } - } + Ok(#(lexer, token)) -> { + case reduce(output, lexer.position, token) { + Ok(output) -> fold_tokens(lexer, output, reduce) + Error(error) -> Error(error) } } - [g, ..] -> Error(Unexpected(g, "[")) - [] -> Ok(toml) + Error(error) -> Error(error) } } -fn parse_array_of_tables( - input: Tokens, -) -> Parsed(#(List(String), Dict(String, Toml))) { - let input = skip_line_whitespace(input) - use key, input <- do(parse_key(input, [])) - use input <- expect(input, "]") - use input <- expect(input, "]") - use table, input <- do(parse_table(input, dict.new())) - Ok(#(#(key, table), input)) -} - -fn parse_table_header(input: Tokens) -> Parsed(List(String)) { - let input = skip_line_whitespace(input) - use key, input <- do(parse_key(input, [])) - use input <- expect(input, "]") - let input = skip_line_whitespace(input) - use input <- expect_end_of_line(input) - Ok(#(key, input)) -} +fn next_token(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "" -> Ok(#(lexer, EndOfFile)) -fn parse_table_and_header( - input: Tokens, -) -> Parsed(#(List(String), Dict(String, Toml))) { - use key, input <- do(parse_table_header(input)) - use table, input <- do(parse_table(input, dict.new())) - Ok(#(#(key, table), input)) -} + "[[" <> src -> Ok(#(advance(lexer, src), DoubleLeftBracketToken)) + "]]" <> src -> Ok(#(advance(lexer, src), DoubleRightBracketToken)) + "\n" <> src -> Ok(#(advance(lexer, src), NewlineToken)) + "[" <> src -> Ok(#(advance(lexer, src), LeftBracketToken)) + "]" <> src -> Ok(#(advance(lexer, src), RightBracketToken)) + "{" <> src -> Ok(#(advance(lexer, src), LeftBraceToken)) + "}" <> src -> Ok(#(advance(lexer, src), RightBraceToken)) + "." <> src -> Ok(#(advance(lexer, src), DotToken)) + "=" <> src -> Ok(#(advance(lexer, src), EqualsToken)) -fn parse_table( - input: Tokens, - toml: Dict(String, Toml), -) -> Parsed(Dict(String, Toml)) { - let input = skip_whitespace(input) - case input { - ["[", ..] | [] -> Ok(#(toml, input)) - _ -> - case parse_key_value(input, toml) { - Ok(#(toml, input)) -> - case skip_line_whitespace(input) { - [] -> Ok(#(toml, [])) - ["\n", ..in] | ["\r\n", ..in] -> parse_table(in, toml) - [g, ..] -> Error(Unexpected(g, "\n")) - } - e -> e - } - } -} + "true" <> src -> Ok(#(advance(lexer, src), BoolToken(True))) + "false" <> src -> Ok(#(advance(lexer, src), BoolToken(False))) -fn parse_key_value( - input: Tokens, - toml: Dict(String, Toml), -) -> Parsed(Dict(String, Toml)) { - use key, input <- do(parse_key(input, [])) - let input = skip_line_whitespace(input) - use input <- expect(input, "=") - let input = skip_line_whitespace(input) - use value, input <- do(parse_value(input)) - case insert(toml, key, value) { - Ok(toml) -> Ok(#(toml, input)) - Error(e) -> Error(e) - } -} + "nan" <> src -> Ok(#(advance(lexer, src), NanToken(None))) + "-nan" <> src -> Ok(#(advance(lexer, src), NanToken(Some(Negative)))) + "+nan" <> src -> Ok(#(advance(lexer, src), NanToken(Some(Positive)))) -fn insert( - table: Dict(String, Toml), - key: List(String), - value: Toml, -) -> Result(Dict(String, Toml), ParseError) { - case insert_loop(table, key, value) { - Ok(table) -> Ok(table) - Error(path) -> Error(KeyAlreadyInUse(path)) - } -} + "inf" <> src -> Ok(#(advance(lexer, src), InfinityToken(None))) + "-inf" <> src -> Ok(#(advance(lexer, src), InfinityToken(Some(Negative)))) + "+inf" <> src -> Ok(#(advance(lexer, src), InfinityToken(Some(Positive)))) -fn insert_loop( - table: Dict(String, Toml), - key: List(String), - value: Toml, -) -> Result(Dict(String, Toml), List(String)) { - case key { - [] -> panic as "unreachable" - [k] -> { - case dict.get(table, k) { - Error(Nil) -> Ok(dict.insert(table, k, value)) - Ok(old) -> merge(table, k, old, value) - } - } - [k, ..key] -> { - case dict.get(table, k) { - Error(Nil) -> { - case insert_loop(dict.new(), key, value) { - Ok(inner) -> Ok(dict.insert(table, k, Table(inner))) - Error(path) -> Error([k, ..path]) - } - } - Ok(ArrayOfTables([inner, ..rest])) -> { - case insert_loop(inner, key, value) { - Ok(inner) -> - Ok(dict.insert(table, k, ArrayOfTables([inner, ..rest]))) - Error(path) -> Error([k, ..path]) - } + "#" <> src -> { + case string.split_once(src, "\n") { + Ok(#(comment, src)) -> { + Ok(#(advance(lexer, src), CommentToken(comment))) } - Ok(Table(inner)) -> { - case insert_loop(inner, key, value) { - Ok(inner) -> Ok(dict.insert(table, k, Table(inner))) - Error(path) -> Error([k, ..path]) - } - } - Ok(_) -> Error([k]) - } - } - } -} - -fn merge( - table: Dict(String, Toml), - key: String, - old: Toml, - new: Toml, -) -> Result(Dict(String, Toml), List(String)) { - case old, new { - // When both are arrays of tables then they are merged together - ArrayOfTables(tables), ArrayOfTables(new) -> - Ok(dict.insert(table, key, ArrayOfTables(list.append(new, tables)))) - - _, _ -> Error([key]) - } -} - -fn expect_end_of_line(input: Tokens, next: fn(Tokens) -> Parsed(a)) -> Parsed(a) { - case input { - ["\n", ..input] -> next(input) - ["\r\n", ..input] -> next(input) - [g, ..] -> Error(Unexpected(g, "\n")) - [] -> Error(Unexpected("EOF", "\n")) - } -} - -fn parse_value(input) -> Parsed(Toml) { - case input { - ["t", "r", "u", "e", ..input] -> Ok(#(Bool(True), input)) - ["f", "a", "l", "s", "e", ..input] -> Ok(#(Bool(False), input)) - - ["n", "a", "n", ..input] -> Ok(#(Nan(Positive), input)) - ["+", "n", "a", "n", ..input] -> Ok(#(Nan(Positive), input)) - ["-", "n", "a", "n", ..input] -> Ok(#(Nan(Negative), input)) - - ["i", "n", "f", ..input] -> Ok(#(Infinity(Positive), input)) - ["+", "i", "n", "f", ..input] -> Ok(#(Infinity(Positive), input)) - ["-", "i", "n", "f", ..input] -> Ok(#(Infinity(Negative), input)) - - ["[", ..input] -> parse_array(input, []) - ["{", ..input] -> parse_inline_table(input, dict.new()) - - ["0", "x", ..input] -> parse_hex(input, 0, Positive) - ["+", "0", "x", ..input] -> parse_hex(input, 0, Positive) - ["-", "0", "x", ..input] -> parse_hex(input, 0, Negative) - - ["0", "o", ..input] -> parse_octal(input, 0, Positive) - ["+", "0", "o", ..input] -> parse_octal(input, 0, Positive) - ["-", "0", "o", ..input] -> parse_octal(input, 0, Negative) - - ["0", "b", ..input] -> parse_binary(input, 0, Positive) - ["+", "0", "b", ..input] -> parse_binary(input, 0, Positive) - ["-", "0", "b", ..input] -> parse_binary(input, 0, Negative) - - ["+", ..input] -> parse_number(input, 0, Positive) - ["-", ..input] -> parse_number(input, 0, Negative) - ["0", ..] - | ["1", ..] - | ["2", ..] - | ["3", ..] - | ["4", ..] - | ["5", ..] - | ["6", ..] - | ["7", ..] - | ["8", ..] - | ["9", ..] -> parse_number(input, 0, Positive) - - ["\"", "\"", "\"", ..input] -> parse_multi_line_string(input, "") - ["\"", ..input] -> parse_string(input, "") - - ["'", "'", "'", ..input] -> parse_multi_line_literal_string(input, "") - ["'", ..input] -> parse_literal_string(input, "") - - [g, ..] -> Error(Unexpected(g, "value")) - [] -> Error(Unexpected("EOF", "value")) - } -} - -fn parse_key(input: Tokens, segments: List(String)) -> Parsed(List(String)) { - use segment, input <- do(parse_key_segment(input)) - let segments = [segment, ..segments] - let input = skip_line_whitespace(input) - - case input { - [".", ..input] -> parse_key(input, segments) - _ -> Ok(#(list.reverse(segments), input)) - } -} - -fn parse_key_segment(input: Tokens) -> Parsed(String) { - let input = skip_line_whitespace(input) - case input { - ["=", ..] -> Error(Unexpected("=", "Key")) - ["\n", ..] -> Error(Unexpected("\n", "Key")) - ["\r\n", ..] -> Error(Unexpected("\r\n", "Key")) - ["[", ..] -> Error(Unexpected("[", "Key")) - ["\"", ..input] -> parse_key_quoted(input, "\"", "") - ["'", ..input] -> parse_key_quoted(input, "'", "") - _ -> parse_key_bare(input, "") - } -} - -fn parse_key_quoted( - input: Tokens, - close: String, - name: String, -) -> Parsed(String) { - case input { - [g, ..input] if g == close -> Ok(#(name, input)) - [g, ..input] -> parse_key_quoted(input, close, name <> g) - [] -> Error(Unexpected("EOF", close)) - } -} - -fn parse_key_bare(input: Tokens, name: String) -> Parsed(String) { - case input { - [" ", ..input] if name != "" -> Ok(#(name, input)) - ["=", ..] if name != "" -> Ok(#(name, input)) - [".", ..] if name != "" -> Ok(#(name, input)) - ["]", ..] if name != "" -> Ok(#(name, input)) - [",", ..] if name != "" -> Error(Unexpected(",", "=")) - ["\n", ..] if name != "" -> Error(Unexpected("\n", "=")) - ["\r\n", ..] if name != "" -> Error(Unexpected("\r\n", "=")) - ["\n", ..] -> Error(Unexpected("\n", "key")) - ["\r\n", ..] -> Error(Unexpected("\r\n", "key")) - ["]", ..] -> Error(Unexpected("]", "key")) - [",", ..] -> Error(Unexpected(",", "key")) - [g, ..input] -> parse_key_bare(input, name <> g) - [] -> Error(Unexpected("EOF", "key")) - } -} - -fn skip_line_whitespace(input: Tokens) -> Tokens { - list.drop_while(input, fn(g) { g == " " || g == "\t" }) -} - -fn skip_whitespace(input: Tokens) -> Tokens { - case input { - [" ", ..input] -> skip_whitespace(input) - ["\t", ..input] -> skip_whitespace(input) - ["\n", ..input] -> skip_whitespace(input) - ["\r\n", ..input] -> skip_whitespace(input) - input -> input - } -} - -type StringState { - // Not currently inside any string - NotInString - // Inside a "..." double-quoted string - InDoubleString - // Inside a """...""" multiline double-quoted string - InMultilineDoubleString - // Inside a '...' single-quoted string - InSingleString - // Inside a '''...''' multiline single-quoted string - InMultilineSingleString -} - -fn drop_comments(input: Tokens, acc: Tokens, state: StringState) -> Tokens { - case input { - ["#", ..input] if state == NotInString -> - input - |> list.drop_while(fn(g) { g != "\n" }) - |> drop_comments(acc, NotInString) - - ["\\", "\"", ..input] if state == InDoubleString -> - drop_comments(input, ["\"", "\\", ..acc], state) - ["\\", "\"", ..input] if state == InMultilineDoubleString -> - drop_comments(input, ["\"", "\\", ..acc], state) - - ["\"", "\"", "\"", ..input] if state == NotInString -> - drop_comments(input, ["\"", "\"", "\"", ..acc], InMultilineDoubleString) - ["\"", "\"", "\"", ..input] if state == InMultilineDoubleString -> - drop_comments(input, ["\"", "\"", "\"", ..acc], NotInString) - - ["\"", ..input] if state == NotInString -> - drop_comments(input, ["\"", ..acc], InDoubleString) - ["\"", ..input] if state == InDoubleString -> - drop_comments(input, ["\"", ..acc], NotInString) - - ["\"", ..input] -> drop_comments(input, ["\"", ..acc], state) - ["'", "'", "'", ..input] if state == NotInString -> - drop_comments(input, ["'", "'", "'", ..acc], InMultilineSingleString) - ["'", "'", "'", ..input] if state == InMultilineSingleString -> - drop_comments(input, ["'", "'", "'", ..acc], NotInString) - ["'", ..input] if state == NotInString -> - drop_comments(input, ["'", ..acc], InSingleString) - ["'", ..input] if state == InSingleString -> - drop_comments(input, ["'", ..acc], NotInString) - ["'", ..input] -> drop_comments(input, ["'", ..acc], state) - - [g, ..input] -> drop_comments(input, [g, ..acc], state) - [] -> list.reverse(acc) - } -} - -fn do( - result: Result(#(a, Tokens), ParseError), - next: fn(a, Tokens) -> Result(b, ParseError), -) -> Result(b, ParseError) { - case result { - Ok(#(a, input)) -> next(a, input) - Error(e) -> Error(e) - } -} - -fn expect( - input: Tokens, - expected: String, - next: fn(Tokens) -> Parsed(a), -) -> Parsed(a) { - case input { - [g, ..input] if g == expected -> next(input) - [g, ..] -> Error(Unexpected(g, expected)) - [] -> Error(Unexpected("EOF", expected)) - } -} - -fn parse_inline_table( - input: Tokens, - properties: Dict(String, Toml), -) -> Parsed(Toml) { - let input = skip_whitespace(input) - case input { - ["}", ..input] -> Ok(#(InlineTable(properties), input)) - _ -> - case parse_inline_table_property(input, properties) { - Ok(#(properties, input)) -> { - let input = skip_whitespace(input) - case input { - ["}", ..input] -> Ok(#(InlineTable(properties), input)) - [",", ..input] -> { - let input = skip_whitespace(input) - parse_inline_table(input, properties) - } - [g, ..] -> Error(Unexpected(g, "}")) - [] -> Error(Unexpected("EOF", "}")) - } - } - Error(e) -> Error(e) - } - } -} - -fn parse_inline_table_property( - input: Tokens, - properties: Dict(String, Toml), -) -> Parsed(Dict(String, Toml)) { - let input = skip_whitespace(input) - use key, input <- do(parse_key(input, [])) - let input = skip_line_whitespace(input) - use input <- expect(input, "=") - let input = skip_line_whitespace(input) - use value, input <- do(parse_value(input)) - case insert(properties, key, value) { - Ok(properties) -> Ok(#(properties, input)) - Error(e) -> Error(e) - } -} - -fn parse_array(input: Tokens, elements: List(Toml)) -> Parsed(Toml) { - let input = skip_whitespace(input) - case input { - ["]", ..input] -> Ok(#(Array(list.reverse(elements)), input)) - _ -> { - use element, input <- do(parse_value(input)) - let elements = [element, ..elements] - let input = skip_whitespace(input) - case input { - ["]", ..input] -> Ok(#(Array(list.reverse(elements)), input)) - [",", ..input] -> { - let input = skip_whitespace(input) - parse_array(input, elements) - } - [g, ..] -> Error(Unexpected(g, "]")) - [] -> Error(Unexpected("EOF", "]")) - } - } - } -} - -fn parse_hex(input: Tokens, number: Int, sign: Sign) -> Parsed(Toml) { - case input { - ["_", ..input] -> parse_hex(input, number, sign) - ["0", ..input] -> parse_hex(input, number * 16 + 0, sign) - ["1", ..input] -> parse_hex(input, number * 16 + 1, sign) - ["2", ..input] -> parse_hex(input, number * 16 + 2, sign) - ["3", ..input] -> parse_hex(input, number * 16 + 3, sign) - ["4", ..input] -> parse_hex(input, number * 16 + 4, sign) - ["5", ..input] -> parse_hex(input, number * 16 + 5, sign) - ["6", ..input] -> parse_hex(input, number * 16 + 6, sign) - ["7", ..input] -> parse_hex(input, number * 16 + 7, sign) - ["8", ..input] -> parse_hex(input, number * 16 + 8, sign) - ["9", ..input] -> parse_hex(input, number * 16 + 9, sign) - ["a", ..input] -> parse_hex(input, number * 16 + 10, sign) - ["b", ..input] -> parse_hex(input, number * 16 + 11, sign) - ["c", ..input] -> parse_hex(input, number * 16 + 12, sign) - ["d", ..input] -> parse_hex(input, number * 16 + 13, sign) - ["e", ..input] -> parse_hex(input, number * 16 + 14, sign) - ["f", ..input] -> parse_hex(input, number * 16 + 15, sign) - ["A", ..input] -> parse_hex(input, number * 16 + 10, sign) - ["B", ..input] -> parse_hex(input, number * 16 + 11, sign) - ["C", ..input] -> parse_hex(input, number * 16 + 12, sign) - ["D", ..input] -> parse_hex(input, number * 16 + 13, sign) - ["E", ..input] -> parse_hex(input, number * 16 + 14, sign) - ["F", ..input] -> parse_hex(input, number * 16 + 15, sign) - - // Anything else and the number is terminated - input -> { - let number = case sign { - Positive -> number - Negative -> -number - } - Ok(#(Int(number), input)) - } - } -} - -fn parse_octal(input: Tokens, number: Int, sign: Sign) -> Parsed(Toml) { - case input { - ["_", ..input] -> parse_octal(input, number, sign) - ["0", ..input] -> parse_octal(input, number * 8 + 0, sign) - ["1", ..input] -> parse_octal(input, number * 8 + 1, sign) - ["2", ..input] -> parse_octal(input, number * 8 + 2, sign) - ["3", ..input] -> parse_octal(input, number * 8 + 3, sign) - ["4", ..input] -> parse_octal(input, number * 8 + 4, sign) - ["5", ..input] -> parse_octal(input, number * 8 + 5, sign) - ["6", ..input] -> parse_octal(input, number * 8 + 6, sign) - ["7", ..input] -> parse_octal(input, number * 8 + 7, sign) - - // Anything else and the number is terminated - input -> { - let number = case sign { - Positive -> number - Negative -> -number - } - Ok(#(Int(number), input)) - } - } -} - -fn parse_binary(input: Tokens, number: Int, sign: Sign) -> Parsed(Toml) { - case input { - ["_", ..input] -> parse_binary(input, number, sign) - ["0", ..input] -> parse_binary(input, number * 2 + 0, sign) - ["1", ..input] -> parse_binary(input, number * 2 + 1, sign) - - // Anything else and the number is terminated - input -> { - let number = case sign { - Positive -> number - Negative -> -number - } - Ok(#(Int(number), input)) - } - } -} - -fn parse_number(input: Tokens, number: Int, sign: Sign) -> Parsed(Toml) { - case input { - ["_", ..input] -> parse_number(input, number, sign) - ["0", ..input] -> parse_number(input, number * 10 + 0, sign) - ["1", ..input] -> parse_number(input, number * 10 + 1, sign) - ["2", ..input] -> parse_number(input, number * 10 + 2, sign) - ["3", ..input] -> parse_number(input, number * 10 + 3, sign) - ["4", ..input] -> parse_number(input, number * 10 + 4, sign) - ["5", ..input] -> parse_number(input, number * 10 + 5, sign) - ["6", ..input] -> parse_number(input, number * 10 + 6, sign) - ["7", ..input] -> parse_number(input, number * 10 + 7, sign) - ["8", ..input] -> parse_number(input, number * 10 + 8, sign) - ["9", ..input] -> parse_number(input, number * 10 + 9, sign) - - ["-", ..input] -> parse_date(input, number) - [":", ..input] if number < 24 -> parse_time_minute(input, number) - - [".", ..input] -> parse_float(input, int.to_float(number), sign, 0.1) - - ["e", "+", ..input] -> - parse_exponent(input, int.to_float(number), sign, 0, Positive) - ["e", "-", ..input] -> - parse_exponent(input, int.to_float(number), sign, 0, Negative) - ["e", ..input] -> - parse_exponent(input, int.to_float(number), sign, 0, Positive) - ["E", "+", ..input] -> - parse_exponent(input, int.to_float(number), sign, 0, Positive) - ["E", "-", ..input] -> - parse_exponent(input, int.to_float(number), sign, 0, Negative) - ["E", ..input] -> - parse_exponent(input, int.to_float(number), sign, 0, Positive) - - // Anything else and the number is terminated - input -> { - let number = case sign { - Positive -> number - Negative -> -number - } - Ok(#(Int(number), input)) - } - } -} - -fn parse_exponent( - input: Tokens, - n: Float, - n_sign: Sign, - ex: Int, - ex_sign: Sign, -) -> Parsed(Toml) { - case input { - ["_", ..input] -> parse_exponent(input, n, n_sign, ex, ex_sign) - ["0", ..input] -> parse_exponent(input, n, n_sign, ex * 10, ex_sign) - ["1", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 1, ex_sign) - ["2", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 2, ex_sign) - ["3", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 3, ex_sign) - ["4", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 4, ex_sign) - ["5", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 5, ex_sign) - ["6", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 6, ex_sign) - ["7", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 7, ex_sign) - ["8", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 8, ex_sign) - ["9", ..input] -> parse_exponent(input, n, n_sign, ex * 10 + 9, ex_sign) - - // Anything else and the number is terminated - input -> { - let number = case n_sign { - Positive -> n - Negative -> n *. -1.0 - } - let exponent = - int.to_float(case ex_sign { - Positive -> ex - Negative -> -ex - }) - let multiplier = case float.power(10.0, exponent) { - Ok(multiplier) -> multiplier - Error(_) -> 1.0 - } - Ok(#(Float(number *. multiplier), input)) - } - } -} - -fn parse_float( - input: Tokens, - number: Float, - sign: Sign, - unit: Float, -) -> Parsed(Toml) { - case input { - ["_", ..input] -> parse_float(input, number, sign, unit) - ["0", ..input] -> parse_float(input, number, sign, unit *. 0.1) - ["1", ..input] -> - parse_float(input, number +. 1.0 *. unit, sign, unit *. 0.1) - ["2", ..input] -> - parse_float(input, number +. 2.0 *. unit, sign, unit *. 0.1) - ["3", ..input] -> - parse_float(input, number +. 3.0 *. unit, sign, unit *. 0.1) - ["4", ..input] -> - parse_float(input, number +. 4.0 *. unit, sign, unit *. 0.1) - ["5", ..input] -> - parse_float(input, number +. 5.0 *. unit, sign, unit *. 0.1) - ["6", ..input] -> - parse_float(input, number +. 6.0 *. unit, sign, unit *. 0.1) - ["7", ..input] -> - parse_float(input, number +. 7.0 *. unit, sign, unit *. 0.1) - ["8", ..input] -> - parse_float(input, number +. 8.0 *. unit, sign, unit *. 0.1) - ["9", ..input] -> - parse_float(input, number +. 9.0 *. unit, sign, unit *. 0.1) - - ["e", "+", ..input] -> parse_exponent(input, number, sign, 0, Positive) - ["e", "-", ..input] -> parse_exponent(input, number, sign, 0, Negative) - ["e", ..input] -> parse_exponent(input, number, sign, 0, Positive) - ["E", "+", ..input] -> parse_exponent(input, number, sign, 0, Positive) - ["E", "-", ..input] -> parse_exponent(input, number, sign, 0, Negative) - ["E", ..input] -> parse_exponent(input, number, sign, 0, Positive) - - // Anything else and the number is terminated - input -> { - let number = case sign { - Positive -> number - Negative -> number *. -1.0 + Error(_) -> todo } - Ok(#(Float(number), input)) } - } -} - -fn parse_string(input: Tokens, string: String) -> Parsed(Toml) { - case input { - ["\"", ..input] -> Ok(#(String(string), input)) - ["\\", "t", ..input] -> parse_string(input, string <> "\t") - ["\\", "e", ..input] -> parse_string(input, string <> "\u{001b}") - ["\\", "b", ..input] -> parse_string(input, string <> "\u{0008}") - ["\\", "n", ..input] -> parse_string(input, string <> "\n") - ["\\", "r", ..input] -> parse_string(input, string <> "\r") - ["\\", "f", ..input] -> parse_string(input, string <> "\f") - ["\\", "\"", ..input] -> parse_string(input, string <> "\"") - ["\\", "\\", ..input] -> parse_string(input, string <> "\\") - [] -> Error(Unexpected("EOF", "\"")) - ["\n", ..] -> Error(Unexpected("\n", "\"")) - ["\r\n", ..] -> Error(Unexpected("\r\n", "\"")) - [g, ..input] -> parse_string(input, string <> g) - } -} - -fn parse_multi_line_string(input: Tokens, string: String) -> Parsed(Toml) { - case input { - ["\"", "\"", "\"", ..input] -> Ok(#(String(string), input)) - ["\\", "\n", ..input] -> - parse_multi_line_string(skip_whitespace(input), string) - ["\\", "\r\n", ..input] -> - parse_multi_line_string(skip_whitespace(input), string) - ["\r\n", ..input] if string == "" -> parse_multi_line_string(input, string) - ["\n", ..input] if string == "" -> parse_multi_line_string(input, string) - ["\r\n", ..input] if string == "" -> parse_multi_line_string(input, string) - ["\\", "t", ..input] -> parse_multi_line_string(input, string <> "\t") - ["\\", "n", ..input] -> parse_multi_line_string(input, string <> "\n") - ["\\", "r", ..input] -> parse_multi_line_string(input, string <> "\r") - ["\\", "\"", ..input] -> parse_multi_line_string(input, string <> "\"") - ["\\", "\\", ..input] -> parse_multi_line_string(input, string <> "\\") - [] -> Error(Unexpected("EOF", "\"")) - [g, ..input] -> parse_multi_line_string(input, string <> g) - } -} - -fn parse_multi_line_literal_string( - input: Tokens, - string: String, -) -> Parsed(Toml) { - case input { - [] -> Error(Unexpected("EOF", "\"")) - ["'", "'", "'", "'", ..] -> Error(Unexpected("''''", "'''")) - ["'", "'", "'", ..input] -> Ok(#(String(string), input)) - ["\n", ..input] if string == "" -> - parse_multi_line_literal_string(input, string) - ["\r\n", ..input] if string == "" -> - parse_multi_line_literal_string(input, string) - [g, ..input] -> parse_multi_line_literal_string(input, string <> g) - } -} - -fn parse_literal_string(input: Tokens, string: String) -> Parsed(Toml) { - case input { - [] -> Error(Unexpected("EOF", "\"")) - ["\n", ..] -> Error(Unexpected("\n", "'")) - ["\r\n", ..] -> Error(Unexpected("\r\n", "'")) - ["'", ..input] -> Ok(#(String(string), input)) - [g, ..input] -> parse_literal_string(input, string <> g) - } -} - -fn reverse_arrays_of_tables(toml: Toml) -> Toml { - case toml { - ArrayOfTables(tables) -> - ArrayOfTables(reverse_arrays_of_tables_array(tables, [])) - - Table(table) -> Table(reverse_arrays_of_tables_table(table)) - - _ -> toml - } -} - -fn reverse_arrays_of_tables_table( - table: Dict(String, Toml), -) -> Dict(String, Toml) { - dict.map_values(table, fn(_, v) { reverse_arrays_of_tables(v) }) -} -fn reverse_arrays_of_tables_array( - array: List(Dict(String, Toml)), - acc: List(Dict(String, Toml)), -) -> List(Dict(String, Toml)) { - case array { - [] -> acc - [first, ..rest] -> { - let first = reverse_arrays_of_tables_table(first) - reverse_arrays_of_tables_array(rest, [first, ..acc]) + src -> { + todo as src } } } - -fn parse_time_minute(input: Tokens, hours: Int) -> Parsed(Toml) { - use minutes, input <- do(parse_number_under_60(input, "minutes")) - use #(seconds, ns), input <- do(parse_time_s_ns(input)) - let time = calendar.TimeOfDay(hours, minutes, seconds, ns) - Ok(#(Time(time), input)) -} - -fn parse_hour_minute(input: Tokens) -> Parsed(#(Int, Int)) { - use hours, input <- do(case input { - ["0", "0", ":", ..input] -> Ok(#(0, input)) - ["0", "1", ":", ..input] -> Ok(#(1, input)) - ["0", "2", ":", ..input] -> Ok(#(2, input)) - ["0", "3", ":", ..input] -> Ok(#(3, input)) - ["0", "4", ":", ..input] -> Ok(#(4, input)) - ["0", "5", ":", ..input] -> Ok(#(5, input)) - ["0", "6", ":", ..input] -> Ok(#(6, input)) - ["0", "7", ":", ..input] -> Ok(#(7, input)) - ["0", "8", ":", ..input] -> Ok(#(8, input)) - ["0", "9", ":", ..input] -> Ok(#(9, input)) - ["1", "0", ":", ..input] -> Ok(#(10, input)) - ["1", "1", ":", ..input] -> Ok(#(11, input)) - ["1", "2", ":", ..input] -> Ok(#(12, input)) - ["1", "3", ":", ..input] -> Ok(#(13, input)) - ["1", "4", ":", ..input] -> Ok(#(14, input)) - ["1", "5", ":", ..input] -> Ok(#(15, input)) - ["1", "6", ":", ..input] -> Ok(#(16, input)) - ["1", "7", ":", ..input] -> Ok(#(17, input)) - ["1", "8", ":", ..input] -> Ok(#(18, input)) - ["1", "9", ":", ..input] -> Ok(#(19, input)) - ["2", "0", ":", ..input] -> Ok(#(20, input)) - ["2", "1", ":", ..input] -> Ok(#(21, input)) - ["2", "2", ":", ..input] -> Ok(#(22, input)) - ["2", "3", ":", ..input] -> Ok(#(23, input)) - [g, ..] -> Error(Unexpected(g, "time")) - [] -> Error(Unexpected("EOF", "time")) - }) - - use minutes, input <- do(parse_number_under_60(input, "minutes")) - Ok(#(#(hours, minutes), input)) -} - -fn parse_time_value(input: Tokens) -> Parsed(calendar.TimeOfDay) { - use #(hours, minutes), input <- do(parse_hour_minute(input)) - use #(seconds, ns), input <- do(parse_time_s_ns(input)) - let time = calendar.TimeOfDay(hours, minutes, seconds, ns) - Ok(#(time, input)) -} - -fn parse_time_s_ns(input: Tokens) -> Parsed(#(Int, Int)) { - case input { - [":", ..input] -> { - use seconds, input <- do(parse_number_under_60(input, "seconds")) - case input { - [".", ..input] -> parse_time_ns(input, seconds, 0, 0) - _ -> Ok(#(#(seconds, 0), input)) - } - } - - _ -> Ok(#(#(0, 0), input)) - } -} - -fn parse_time_ns( - input: Tokens, - seconds: Int, - ns: Int, - digits_count: Int, -) -> Parsed(#(Int, Int)) { - case input { - ["0", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 0, digits_count + 1) - ["1", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 1, digits_count + 1) - ["2", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 2, digits_count + 1) - ["3", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 3, digits_count + 1) - ["4", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 4, digits_count + 1) - ["5", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 5, digits_count + 1) - ["6", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 6, digits_count + 1) - ["7", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 7, digits_count + 1) - ["8", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 8, digits_count + 1) - ["9", ..input] if digits_count < 9 -> - parse_time_ns(input, seconds, ns * 10 + 9, digits_count + 1) - - // Anything else and the number is terminated - _ -> { - let exponent = int.to_float(9 - digits_count) - let assert Ok(multiplier) = float.power(10.0, exponent) - Ok(#(#(seconds, ns * float.truncate(multiplier)), input)) - } - } -} - -fn parse_number_under_60(input: Tokens, expected: String) -> Parsed(Int) { - case input { - ["0", "0", ..input] -> Ok(#(0, input)) - ["0", "1", ..input] -> Ok(#(1, input)) - ["0", "2", ..input] -> Ok(#(2, input)) - ["0", "3", ..input] -> Ok(#(3, input)) - ["0", "4", ..input] -> Ok(#(4, input)) - ["0", "5", ..input] -> Ok(#(5, input)) - ["0", "6", ..input] -> Ok(#(6, input)) - ["0", "7", ..input] -> Ok(#(7, input)) - ["0", "8", ..input] -> Ok(#(8, input)) - ["0", "9", ..input] -> Ok(#(9, input)) - ["1", "0", ..input] -> Ok(#(10, input)) - ["1", "1", ..input] -> Ok(#(11, input)) - ["1", "2", ..input] -> Ok(#(12, input)) - ["1", "3", ..input] -> Ok(#(13, input)) - ["1", "4", ..input] -> Ok(#(14, input)) - ["1", "5", ..input] -> Ok(#(15, input)) - ["1", "6", ..input] -> Ok(#(16, input)) - ["1", "7", ..input] -> Ok(#(17, input)) - ["1", "8", ..input] -> Ok(#(18, input)) - ["1", "9", ..input] -> Ok(#(19, input)) - ["2", "0", ..input] -> Ok(#(20, input)) - ["2", "1", ..input] -> Ok(#(21, input)) - ["2", "2", ..input] -> Ok(#(22, input)) - ["2", "3", ..input] -> Ok(#(23, input)) - ["2", "4", ..input] -> Ok(#(24, input)) - ["2", "5", ..input] -> Ok(#(25, input)) - ["2", "6", ..input] -> Ok(#(26, input)) - ["2", "7", ..input] -> Ok(#(27, input)) - ["2", "8", ..input] -> Ok(#(28, input)) - ["2", "9", ..input] -> Ok(#(29, input)) - ["3", "0", ..input] -> Ok(#(30, input)) - ["3", "1", ..input] -> Ok(#(31, input)) - ["3", "2", ..input] -> Ok(#(32, input)) - ["3", "3", ..input] -> Ok(#(33, input)) - ["3", "4", ..input] -> Ok(#(34, input)) - ["3", "5", ..input] -> Ok(#(35, input)) - ["3", "6", ..input] -> Ok(#(36, input)) - ["3", "7", ..input] -> Ok(#(37, input)) - ["3", "8", ..input] -> Ok(#(38, input)) - ["3", "9", ..input] -> Ok(#(39, input)) - ["4", "0", ..input] -> Ok(#(40, input)) - ["4", "1", ..input] -> Ok(#(41, input)) - ["4", "2", ..input] -> Ok(#(42, input)) - ["4", "3", ..input] -> Ok(#(43, input)) - ["4", "4", ..input] -> Ok(#(44, input)) - ["4", "5", ..input] -> Ok(#(45, input)) - ["4", "6", ..input] -> Ok(#(46, input)) - ["4", "7", ..input] -> Ok(#(47, input)) - ["4", "8", ..input] -> Ok(#(48, input)) - ["4", "9", ..input] -> Ok(#(49, input)) - ["5", "0", ..input] -> Ok(#(50, input)) - ["5", "1", ..input] -> Ok(#(51, input)) - ["5", "2", ..input] -> Ok(#(52, input)) - ["5", "3", ..input] -> Ok(#(53, input)) - ["5", "4", ..input] -> Ok(#(54, input)) - ["5", "5", ..input] -> Ok(#(55, input)) - ["5", "6", ..input] -> Ok(#(56, input)) - ["5", "7", ..input] -> Ok(#(57, input)) - ["5", "8", ..input] -> Ok(#(58, input)) - ["5", "9", ..input] -> Ok(#(59, input)) - - [g, ..] -> Error(Unexpected(g, expected)) - [] -> Error(Unexpected("EOF", expected)) - } -} - -fn parse_date(input: Tokens, year: Int) -> Parsed(Toml) { - case input { - ["0", "1", "-", ..input] -> parse_date_day(input, year, calendar.January) - ["0", "2", "-", ..input] -> parse_date_day(input, year, calendar.February) - ["0", "3", "-", ..input] -> parse_date_day(input, year, calendar.March) - ["0", "4", "-", ..input] -> parse_date_day(input, year, calendar.April) - ["0", "5", "-", ..input] -> parse_date_day(input, year, calendar.May) - ["0", "6", "-", ..input] -> parse_date_day(input, year, calendar.June) - ["0", "7", "-", ..input] -> parse_date_day(input, year, calendar.July) - ["0", "8", "-", ..input] -> parse_date_day(input, year, calendar.August) - ["0", "9", "-", ..input] -> parse_date_day(input, year, calendar.September) - ["1", "0", "-", ..input] -> parse_date_day(input, year, calendar.October) - ["1", "1", "-", ..input] -> parse_date_day(input, year, calendar.November) - ["1", "2", "-", ..input] -> parse_date_day(input, year, calendar.December) - - [g, ..] -> Error(Unexpected(g, "date month")) - [] -> Error(Unexpected("EOF", "date month")) - } -} - -fn parse_date_day( - input: Tokens, - year: Int, - month: calendar.Month, -) -> Parsed(Toml) { - case input { - ["0", "1", ..input] -> parse_date_end(input, year, month, 1) - ["0", "2", ..input] -> parse_date_end(input, year, month, 2) - ["0", "3", ..input] -> parse_date_end(input, year, month, 3) - ["0", "4", ..input] -> parse_date_end(input, year, month, 4) - ["0", "5", ..input] -> parse_date_end(input, year, month, 5) - ["0", "6", ..input] -> parse_date_end(input, year, month, 6) - ["0", "7", ..input] -> parse_date_end(input, year, month, 7) - ["0", "8", ..input] -> parse_date_end(input, year, month, 8) - ["0", "9", ..input] -> parse_date_end(input, year, month, 9) - ["1", "0", ..input] -> parse_date_end(input, year, month, 10) - ["1", "1", ..input] -> parse_date_end(input, year, month, 11) - ["1", "2", ..input] -> parse_date_end(input, year, month, 12) - ["1", "3", ..input] -> parse_date_end(input, year, month, 13) - ["1", "4", ..input] -> parse_date_end(input, year, month, 14) - ["1", "5", ..input] -> parse_date_end(input, year, month, 15) - ["1", "6", ..input] -> parse_date_end(input, year, month, 16) - ["1", "7", ..input] -> parse_date_end(input, year, month, 17) - ["1", "8", ..input] -> parse_date_end(input, year, month, 18) - ["1", "9", ..input] -> parse_date_end(input, year, month, 19) - ["2", "0", ..input] -> parse_date_end(input, year, month, 20) - ["2", "1", ..input] -> parse_date_end(input, year, month, 21) - ["2", "2", ..input] -> parse_date_end(input, year, month, 22) - ["2", "3", ..input] -> parse_date_end(input, year, month, 23) - ["2", "4", ..input] -> parse_date_end(input, year, month, 24) - ["2", "5", ..input] -> parse_date_end(input, year, month, 25) - ["2", "6", ..input] -> parse_date_end(input, year, month, 26) - ["2", "7", ..input] -> parse_date_end(input, year, month, 27) - ["2", "8", ..input] -> parse_date_end(input, year, month, 28) - ["2", "9", ..input] -> parse_date_end(input, year, month, 29) - ["3", "0", ..input] -> parse_date_end(input, year, month, 30) - ["3", "1", ..input] -> parse_date_end(input, year, month, 31) - - [g, ..] -> Error(Unexpected(g, "date day")) - [] -> Error(Unexpected("EOF", "date day")) - } -} - -fn parse_date_end( - input: Tokens, - year: Int, - month: calendar.Month, - day: Int, -) -> Parsed(Toml) { - let date = calendar.Date(year, month, day) - case input { - [" ", ..input] | ["T", ..input] -> { - use time, input <- do(parse_time_value(input)) - use offset, input <- do(parse_offset(input)) - Ok(#(DateTime(date, time, offset), input)) - } - - _ -> Ok(#(Date(date), input)) - } -} - -fn parse_offset(input: Tokens) -> Parsed(Offset) { - case input { - ["Z", ..input] -> Ok(#(Offset(calendar.utc_offset), input)) - ["+", ..input] -> parse_offset_hours(input, Positive) - ["-", ..input] -> parse_offset_hours(input, Negative) - _ -> Ok(#(Local, input)) - } -} - -fn parse_offset_hours(input: Tokens, sign: Sign) -> Parsed(Offset) { - use #(hours, minutes), input <- do(parse_hour_minute(input)) - let duration = case sign { - Positive -> duration.add(duration.hours(hours), duration.minutes(minutes)) - Negative -> duration.add(duration.hours(-hours), duration.minutes(-minutes)) - } - Ok(#(Offset(duration), input)) -} - -/// Get a int from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_int(Int(1)) -/// // -> Ok(1) -/// ``` -/// -/// ```gleam -/// as_int(Float(1.4)) -/// // -> Error(WrongType([], "Int", "Float")) -/// ``` -pub fn as_int(toml: Toml) -> Result(Int, GetError) { - case toml { - Int(f) -> Ok(f) - other -> Error(WrongType([], "Int", classify(other))) - } -} - -/// Get a float from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_float(Float(1.5)) -/// // -> Ok(1.5) -/// ``` -/// -/// ```gleam -/// as_float(Int(1)) -/// // -> Error(WrongType([], "Float", "Int")) -/// ``` -pub fn as_float(toml: Toml) -> Result(Float, GetError) { - case toml { - Float(f) -> Ok(f) - other -> Error(WrongType([], "Float", classify(other))) - } -} - -/// Get a bool from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_bool(Bool(true)) -/// // -> Ok(true) -/// ``` -/// -/// ```gleam -/// as_bool(Int(1)) -/// // -> Error(WrongType([], "Bool", "Int")) -/// ``` -pub fn as_bool(toml: Toml) -> Result(Bool, GetError) { - case toml { - Bool(b) -> Ok(b) - other -> Error(WrongType([], "Bool", classify(other))) - } -} - -/// Get a string from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_string(String("hello")) -/// // -> Ok("hello") -/// ``` -/// -/// ```gleam -/// as_string(Int(1)) -/// // -> Error(WrongType([], "String", "Int")) -/// ``` -pub fn as_string(toml: Toml) -> Result(String, GetError) { - case toml { - String(s) -> Ok(s) - other -> Error(WrongType([], "String", classify(other))) - } -} - -/// Get a date from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_date(Date(date)) -/// // -> Ok(date) -/// ``` -/// -/// ```gleam -/// as_date(Int(1)) -/// // -> Error(WrongType([], "Date", "Int")) -/// ``` -pub fn as_date(toml: Toml) -> Result(calendar.Date, GetError) { - case toml { - Date(d) -> Ok(d) - other -> Error(WrongType([], "Date", classify(other))) - } -} - -/// Get a time from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_time_of_day(Time(time)) -/// // -> Ok(time) -/// ``` -/// -/// ```gleam -/// as_time_of_day(Int(1)) -/// // -> Error(WrongType([], "Time", "Int")) -/// ``` -pub fn as_time_of_day(toml: Toml) -> Result(calendar.TimeOfDay, GetError) { - case toml { - Time(t) -> Ok(t) - other -> Error(WrongType([], "Time", classify(other))) - } -} - -/// Get an unambiguous time from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_timestamp(Int(1)) -/// // -> Error(WrongType([], "DateTime with offset", "Int")) -/// ``` -pub fn as_timestamp(toml: Toml) -> Result(timestamp.Timestamp, GetError) { - case toml { - DateTime(date:, time:, offset: Offset(offset)) -> - Ok(timestamp.from_calendar(date, time, offset)) - - other -> Error(WrongType([], "DateTime with offset", classify(other))) - } -} - -/// Get a datetime from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_calendar_time(DateTime(datetime)) -/// // -> Ok(datetime) -/// ``` -/// -/// ```gleam -/// as_calendar_time(Int(1)) -/// // -> Error(WrongType([], "DateTime", "Int")) -/// ``` -pub fn as_calendar_time( - toml: Toml, -) -> Result(#(calendar.Date, calendar.TimeOfDay, Offset), GetError) { - case toml { - DateTime(d, t, o) -> Ok(#(d, t, o)) - other -> Error(WrongType([], "DateTime", classify(other))) - } -} - -/// Get an array from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_array(Array([])) -/// // -> Ok([]) -/// ``` -/// -/// ```gleam -/// as_array(Int(1)) -/// // -> Error(WrongType([], "Array", "Int")) -/// ``` -pub fn as_array(toml: Toml) -> Result(List(Toml), GetError) { - case toml { - Array(arr) -> Ok(arr) - other -> Error(WrongType([], "Array", classify(other))) - } -} - -/// Get a table from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_table(Table(dict.new())) -/// // -> Ok(dict.new()) -/// ``` -/// -/// ```gleam -/// as_table(Int(1)) -/// // -> Error(WrongType([], "Table", "Int")) -/// ``` -pub fn as_table(toml: Toml) -> Result(Dict(String, Toml), GetError) { - case toml { - Table(tbl) -> Ok(tbl) - InlineTable(tbl) -> Ok(tbl) - other -> Error(WrongType([], "Table", classify(other))) - } -} - -/// Get a number (int or float) from a TOML document. -/// -/// ## Examples -/// -/// ```gleam -/// as_number(Int(1)) -/// // -> Ok(NumberInt(1)) -/// ``` -/// -/// ```gleam -/// as_number(Float(1.5)) -/// // -> Ok(NumberFloat(1.5)) -/// ``` -/// -/// ```gleam -/// as_number(Bool(true)) -/// // -> Error(WrongType([], "Number", "Bool")) -/// ``` -pub fn as_number(toml: Toml) -> Result(Number, GetError) { - case toml { - Int(x) -> Ok(NumberInt(x)) - Float(x) -> Ok(NumberFloat(x)) - Nan(x) -> Ok(NumberNan(x)) - Infinity(x) -> Ok(NumberInfinity(x)) - other -> Error(WrongType([], "Number", classify(other))) - } -} diff --git a/test/tom/tokenise_test.gleam b/test/tom/tokenise_test.gleam new file mode 100644 index 0000000..8645247 --- /dev/null +++ b/test/tom/tokenise_test.gleam @@ -0,0 +1,85 @@ +import gleam/option.{None, Some} +import tom.{ + BareKeyToken, BoolToken, CommentToken, DotToken, DoubleLeftBracketToken, + DoubleRightBracketToken, EndOfFile, EqualsToken, InfinityToken, IntToken, + LeftBraceToken, LeftBracketToken, NanToken, Negative, NewlineToken, Positive, + RightBraceToken, RightBracketToken, WhitespaceToken, +} + +pub fn empty_test() { + assert tom.to_tokens("") == Ok([EndOfFile]) +} + +pub fn equal_test() { + assert tom.to_tokens("=") == Ok([EqualsToken, EndOfFile]) +} + +pub fn dot_test() { + assert tom.to_tokens(".") == Ok([DotToken, EndOfFile]) +} + +pub fn braces_test() { + assert tom.to_tokens("{}") == Ok([LeftBraceToken, RightBraceToken, EndOfFile]) +} + +pub fn brackets_test() { + assert tom.to_tokens("[]") + == Ok([LeftBracketToken, RightBracketToken, EndOfFile]) +} + +pub fn double_brackets_test() { + assert tom.to_tokens("[[]]") + == Ok([DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFile]) +} + +pub fn true_test() { + assert tom.to_tokens("true") == Ok([BoolToken(True), EndOfFile]) +} + +pub fn false_test() { + assert tom.to_tokens("false") == Ok([BoolToken(False), EndOfFile]) +} + +pub fn nan_test() { + assert tom.to_tokens("nan") == Ok([NanToken(None), EndOfFile]) +} + +pub fn nan_positive_test() { + assert tom.to_tokens("+nan") == Ok([NanToken(Some(Positive)), EndOfFile]) +} + +pub fn nan_negative_test() { + assert tom.to_tokens("-nan") == Ok([NanToken(Some(Negative)), EndOfFile]) +} + +pub fn inf_test() { + assert tom.to_tokens("inf") == Ok([InfinityToken(None), EndOfFile]) +} + +pub fn inf_positive_test() { + assert tom.to_tokens("+inf") == Ok([InfinityToken(Some(Positive)), EndOfFile]) +} + +pub fn inf_negative_test() { + assert tom.to_tokens("-inf") == Ok([InfinityToken(Some(Negative)), EndOfFile]) +} + +pub fn newline_test() { + assert tom.to_tokens("\n\n\n") + == Ok([NewlineToken, NewlineToken, NewlineToken, EndOfFile]) +} + +pub fn comment_test() { + assert tom.to_tokens("# Hello, world!\n\n") + == Ok([CommentToken(" Hello, world!"), NewlineToken, EndOfFile]) +} +// pub fn key_value_with_whitespace_tokenises_test() { +// assert tom2.to_tokens("answer = 42") +// == Ok([ +// BareKeyToken("answer"), +// WhitespaceToken(" "), +// EqualsToken, +// WhitespaceToken(" "), +// IntToken("42", 42), +// ]) +// } diff --git a/test/tom_test.gleam b/test/tom_test.gleam index 935e7a1..902c4da 100644 --- a/test/tom_test.gleam +++ b/test/tom_test.gleam @@ -1,1066 +1,5 @@ -import gleam/dict -import gleam/result -import gleam/time/calendar -import gleam/time/duration -import gleam/time/timestamp import gleeunit -import tom -pub fn main() { +pub fn main() -> Nil { gleeunit.main() } - -pub fn parse_empty_test() { - assert tom.parse("") == Ok(dict.from_list([])) -} - -pub fn parse_spaces_test() { - assert tom.parse(" ") == Ok(dict.from_list([])) -} - -pub fn parse_newline_test() { - assert tom.parse("\n") == Ok(dict.from_list([])) -} - -pub fn parse_crlf_test() { - assert tom.parse("\r\n") == Ok(dict.from_list([])) -} - -pub fn parse_quoted_key_test() { - let expected = dict.from_list([#(" ", tom.Bool(True))]) - assert tom.parse("\" \" = true\n") == Ok(expected) -} - -pub fn parse_single_key_test() { - let expected = dict.from_list([#("", tom.Bool(True))]) - assert tom.parse("'' = true\n") == Ok(expected) -} - -pub fn parse_true_test() { - let expected = dict.from_list([#("cool", tom.Bool(True))]) - assert tom.parse("cool = true\n") == Ok(expected) -} - -pub fn parse_false_test() { - let expected = dict.from_list([#("cool", tom.Bool(False))]) - assert tom.parse("cool = false\n") == Ok(expected) -} - -pub fn parse_unicode_key_test() { - let expected = dict.from_list([#("பெண்", tom.Bool(False))]) - assert tom.parse("பெண் = false\n") == Ok(expected) -} - -pub fn parse_int_test() { - let expected = dict.from_list([#("it", tom.Int(1))]) - assert tom.parse("it = 1\n") == Ok(expected) -} - -pub fn parse_int_underscored_test() { - let expected = dict.from_list([#("it", tom.Int(1_000_009))]) - assert tom.parse("it = 1_000_0__0_9\n") == Ok(expected) -} - -pub fn parse_int_positive_test() { - let expected = dict.from_list([#("it", tom.Int(234))]) - assert tom.parse("it = +234\n") == Ok(expected) -} - -pub fn parse_int_negative_test() { - let expected = dict.from_list([#("it", tom.Int(-234))]) - assert tom.parse("it = -234\n") == Ok(expected) -} - -pub fn parse_string_test() { - let expected = dict.from_list([#("hello", tom.String("Joe"))]) - assert tom.parse("hello = \"Joe\"\n") == Ok(expected) -} - -pub fn parse_string_escaped_quote_test() { - let expected = dict.from_list([#("hello", tom.String("\""))]) - assert tom.parse("hello = \"\\\"\"\n") == Ok(expected) -} - -pub fn parse_string_tab_test() { - let expected = dict.from_list([#("hello", tom.String("\t"))]) - assert tom.parse("hello = \"\\t\"\n") == Ok(expected) -} - -pub fn parse_string_newline_test() { - let expected = dict.from_list([#("hello", tom.String("\n"))]) - assert tom.parse("hello = \"\\n\"\n") == Ok(expected) -} - -pub fn parse_string_linefeed_test() { - let expected = dict.from_list([#("hello", tom.String("\r"))]) - assert tom.parse("hello = \"\\r\"\n") == Ok(expected) -} - -pub fn parse_escaped_slash_test() { - let expected = dict.from_list([#("hello", tom.String("\\"))]) - assert tom.parse("hello = \"\\\\\"\n") == Ok(expected) -} - -pub fn parse_float_test() { - let expected = dict.from_list([#("it", tom.Float(1.0))]) - assert tom.parse("it = 1.0\n") == Ok(expected) -} - -pub fn parse_bigger_float_test() { - let expected = dict.from_list([#("it", tom.Float(123_456_789.9876))]) - assert tom.parse("it = 123456789.9876\n") == Ok(expected) -} - -pub fn parse_multi_segment_key_test() { - let expected = - dict.from_list([ - #( - "one", - tom.Table( - dict.from_list([ - #("two", tom.Table(dict.from_list([#("three", tom.Bool(True))]))), - ]), - ), - ), - ]) - assert tom.parse("one.two.three = true\n") == Ok(expected) -} - -pub fn parse_multi_segment_key_with_spaeces_test() { - let expected = - dict.from_list([ - #( - "one", - tom.Table( - dict.from_list([ - #("two", tom.Table(dict.from_list([#("three", tom.Bool(True))]))), - ]), - ), - ), - ]) - assert tom.parse("one . two . three = true\n") == Ok(expected) -} - -pub fn parse_multi_segment_key_quotes_test() { - let expected = - dict.from_list([ - #( - "1", - tom.Table( - dict.from_list([ - #("two", tom.Table(dict.from_list([#("3", tom.Bool(True))]))), - ]), - ), - ), - ]) - assert tom.parse("\"1\".two.\"3\" = true\n") == Ok(expected) -} - -pub fn parse_multiple_keys_test() { - let expected = dict.from_list([#("a", tom.Int(1)), #("b", tom.Int(2))]) - assert tom.parse("a = 1\nb = 2\n") == Ok(expected) -} - -pub fn parse_duplicate_key_test() { - assert tom.parse("a = 1\na = 2\n") == Error(tom.KeyAlreadyInUse(["a"])) -} - -pub fn parse_conflicting_keys_test() { - assert tom.parse("a = 1\na.b = 2\n") == Error(tom.KeyAlreadyInUse(["a"])) -} - -pub fn parse_empty_array_test() { - let expected = dict.from_list([#("a", tom.Array([]))]) - assert tom.parse("a = []\n") == Ok(expected) -} - -pub fn parse_array_test() { - let expected = dict.from_list([#("a", tom.Array([tom.Int(1), tom.Int(2)]))]) - assert tom.parse("a = [1, 2]\n") == Ok(expected) -} - -pub fn parse_multi_line_array_test() { - let expected = dict.from_list([#("a", tom.Array([tom.Int(1), tom.Int(2)]))]) - assert tom.parse("a = [\n 1 \n ,\n 2,\n]\n") == Ok(expected) -} - -pub fn parse_table_test() { - let expected = dict.from_list([#("a", tom.Table(dict.from_list([])))]) - assert tom.parse("[a]\n") == Ok(expected) -} - -pub fn parse_table_with_values_test() { - let expected = - dict.from_list([ - #( - "a", - tom.Table( - dict.from_list([ - #("a", tom.Int(1)), - #("b", tom.Table(dict.from_list([#("c", tom.Int(2))]))), - ]), - ), - ), - ]) - assert tom.parse( - "[a] -a = 1 -b.c = 2 -", - ) - == Ok(expected) -} - -pub fn parse_table_with_values_before_test() { - let expected = - dict.from_list([ - #("name", tom.String("Joe")), - #("size", tom.Int(123)), - #( - "a", - tom.Table( - dict.from_list([ - #("a", tom.Int(1)), - #("b", tom.Table(dict.from_list([#("c", tom.Int(2))]))), - ]), - ), - ), - ]) - assert tom.parse( - "name = \"Joe\" -size = 123 - -[a] -a = 1 -b.c = 2 -", - ) - == Ok(expected) -} - -pub fn parse_multiple_tables_test() { - let expected = - dict.from_list([ - #("name", tom.String("Joe")), - #("size", tom.Int(123)), - #( - "a", - tom.Table( - dict.from_list([ - #("a", tom.Int(1)), - #("b", tom.Table(dict.from_list([#("c", tom.Int(2))]))), - ]), - ), - ), - #("b", tom.Table(dict.from_list([#("a", tom.Int(1))]))), - ]) - assert tom.parse( - "name = \"Joe\" -size = 123 - -[a] -a = 1 -b.c = 2 - -[b] -a = 1 -", - ) - == Ok(expected) -} - -pub fn parse_inline_table_empty_test() { - let expected = dict.from_list([#("a", tom.InlineTable(dict.from_list([])))]) - assert tom.parse("a = {}\n") == Ok(expected) -} - -pub fn parse_inline_table_test() { - let expected = - dict.from_list([ - #( - "a", - tom.InlineTable( - dict.from_list([ - #("a", tom.Int(1)), - #("b", tom.Table(dict.from_list([#("c", tom.Int(2))]))), - ]), - ), - ), - ]) - assert tom.parse( - "a = { - a = 1, - b.c = 2 -} -", - ) - == Ok(expected) -} - -pub fn parse_inline_trailing_comma_table_test() { - let expected = - dict.from_list([ - #( - "a", - tom.InlineTable( - dict.from_list([ - #("a", tom.Int(1)), - #("b", tom.Table(dict.from_list([#("c", tom.Int(2))]))), - ]), - ), - ), - ]) - assert tom.parse( - "a = { - a = 1, - b.c = 2, -} -", - ) - == Ok(expected) -} - -pub fn parse_invalid_newline_in_string_test() { - assert tom.parse("a = \"\n\"") == Error(tom.Unexpected("\n", "\"")) -} - -pub fn parse_invalid_newline_windows_in_string_test() { - assert tom.parse("a = \"\r\n\"") == Error(tom.Unexpected("\r\n", "\"")) -} - -pub fn parse_array_of_tables_empty_test() { - let expected = - dict.from_list([ - #( - "a", - tom.ArrayOfTables([ - dict.from_list([]), - dict.from_list([]), - dict.from_list([]), - ]), - ), - ]) - assert tom.parse( - "[[a]] -[[a]] -[[a]] -", - ) - == Ok(expected) -} - -pub fn parse_array_of_tables_nonempty_test() { - let expected = - dict.from_list([ - #( - "a", - tom.ArrayOfTables([ - dict.from_list([#("a", tom.Int(1))]), - dict.from_list([#("a", tom.Int(2))]), - dict.from_list([#("a", tom.Int(3))]), - ]), - ), - ]) - assert tom.parse( - "[[a]] -a = 1 - -[[a]] -a = 2 - -[[a]] -a = 3 -", - ) - == Ok(expected) -} - -pub fn parse_array_of_tables_with_subtable_test() { - let expected = - dict.from_list([ - #( - "fruits", - tom.ArrayOfTables([ - dict.from_list([]), - dict.from_list([ - #("name", tom.String("apple")), - #( - "physical", - tom.Table( - dict.from_list([ - #("color", tom.String("red")), - #("shape", tom.String("round")), - ]), - ), - ), - ]), - ]), - ), - ]) - assert tom.parse( - "[[fruits]] - -[[fruits]] -name = \"apple\" - -[fruits.physical] # subtable -color = \"red\" -shape = \"round\" -", - ) - == Ok(expected) -} - -pub fn parse_single_quote_string_test() { - let expected = dict.from_list([#("a", tom.String("\\n"))]) - assert tom.parse("a = '\\n'\n") == Ok(expected) -} - -pub fn parse_multi_line_string_test() { - let expected = dict.from_list([#("a", tom.String("hello\nworld"))]) - assert tom.parse( - "a = \"\"\" -hello -world\"\"\" -", - ) - == Ok(expected) -} - -pub fn parse_multi_line_single_quote_string_test() { - let expected = dict.from_list([#("a", tom.String("hello\\n\nworld"))]) - assert tom.parse( - "a = ''' -hello\\n -world''' -", - ) - == Ok(expected) -} - -pub fn parse_multi_line_single_quote_string_too_many_quotes_test() { - assert tom.parse( - "a = ''' -'''' -''' -", - ) - == Error(tom.Unexpected("''''", "'''")) -} - -pub fn parse_multi_line_literal_string_with_hash_test() { - let expected = - dict.from_list([ - #( - "a", - tom.String( - "This string contains a #hash character\nand more text after it\n", - ), - ), - ]) - assert tom.parse( - "a = ''' -This string contains a #hash character -and more text after it -''' -", - ) - == Ok(expected) -} - -pub fn parse_multi_line_string_escape_newline_test() { - let expected = - dict.from_list([ - #("a", tom.String("The quick brown fox jumps over the lazy dog.")), - ]) - assert tom.parse( - "a = \"\"\" -The quick brown \\ - - - fox jumps over \\ - the lazy dog.\"\"\" -", - ) - == Ok(expected) -} - -pub fn parse_multi_line_string_escape_newline_windows_test() { - let expected = - dict.from_list([ - #("a", tom.String("The quick brown fox jumps over the lazy dog.")), - ]) - assert tom.parse( - "a = \"\"\" -The quick brown \\\r\n - - - fox jumps over \\\r\n - the lazy dog.\"\"\" -", - ) - == Ok(expected) -} - -pub fn parse_nan_test() { - let expected = dict.from_list([#("a", tom.Nan(tom.Positive))]) - assert tom.parse("a = nan\n") == Ok(expected) -} - -pub fn parse_positive_nan_test() { - let expected = dict.from_list([#("a", tom.Nan(tom.Positive))]) - assert tom.parse("a = +nan\n") == Ok(expected) -} - -pub fn parse_negative_nan_test() { - let expected = dict.from_list([#("a", tom.Nan(tom.Negative))]) - assert tom.parse("a = -nan\n") == Ok(expected) -} - -pub fn parse_infinity_test() { - let expected = dict.from_list([#("a", tom.Infinity(tom.Positive))]) - assert tom.parse("a = inf\n") == Ok(expected) -} - -pub fn parse_positive_infinity_test() { - let expected = dict.from_list([#("a", tom.Infinity(tom.Positive))]) - assert tom.parse("a = +inf\n") == Ok(expected) -} - -pub fn parse_negative_infinity_test() { - let expected = dict.from_list([#("a", tom.Infinity(tom.Negative))]) - assert tom.parse("a = -inf\n") == Ok(expected) -} - -pub fn parse_write_to_key_that_does_not_exist_test() { - let expected = - dict.from_list([ - #("apple", tom.Table(dict.from_list([#("smooth", tom.Bool(True))]))), - ]) - assert tom.parse("apple.smooth = true\n") == Ok(expected) -} - -pub fn parse_binary_test() { - let expected = dict.from_list([#("a", tom.Int(0b101010))]) - assert tom.parse("a = 0b101010\n") == Ok(expected) -} - -pub fn parse_binary_positive_test() { - let expected = dict.from_list([#("a", tom.Int(0b101010))]) - assert tom.parse("a = +0b101010\n") == Ok(expected) -} - -pub fn parse_binary_negative_test() { - let expected = dict.from_list([#("a", tom.Int(0b101010 * -1))]) - assert tom.parse("a = -0b101010\n") == Ok(expected) -} - -pub fn parse_binary_underscores_test() { - let expected = dict.from_list([#("a", tom.Int(0b101010))]) - assert tom.parse("a = 0b1__010___1_0\n") == Ok(expected) -} - -pub fn parse_octal_test() { - let expected = dict.from_list([#("a", tom.Int(0o1234567))]) - assert tom.parse("a = 0o1234567\n") == Ok(expected) -} - -pub fn parse_octal_positive_test() { - let expected = dict.from_list([#("a", tom.Int(0o1234567))]) - assert tom.parse("a = +0o1234567\n") == Ok(expected) -} - -pub fn parse_octal_negative_test() { - let expected = dict.from_list([#("a", tom.Int(0o1234567 * -1))]) - assert tom.parse("a = -0o1234567\n") == Ok(expected) -} - -pub fn parse_octal_underscores_test() { - let expected = dict.from_list([#("a", tom.Int(0o1234567))]) - assert tom.parse("a = 0o1_23_45__6_7\n") == Ok(expected) -} - -pub fn parse_hex_test() { - let expected = dict.from_list([#("a", tom.Int(0xdeadbeef))]) - assert tom.parse("a = 0xdeadbeef\n") == Ok(expected) -} - -pub fn parse_hex_positive_test() { - let expected = dict.from_list([#("a", tom.Int(0xdeadbeef))]) - assert tom.parse("a = +0xdeadbeef\n") == Ok(expected) -} - -pub fn parse_hex_negative_test() { - let expected = dict.from_list([#("a", tom.Int(0xdeadbeef * -1))]) - assert tom.parse("a = -0xdeadbeef\n") == Ok(expected) -} - -pub fn parse_hex_underscores_test() { - let expected = dict.from_list([#("a", tom.Int(0xdeadbeef))]) - assert tom.parse("a = 0xd_e_a_d__b___e____e______f\n") == Ok(expected) -} - -pub fn parse_hex_uppercase_test() { - let expected = dict.from_list([#("a", tom.Int(0xdeadbeef))]) - assert tom.parse("a = +0xDEADBEEF\n") == Ok(expected) -} - -pub fn parse_float_exponent_test() { - let expected = dict.from_list([#("a", tom.Float(1.0e6))]) - assert tom.parse("a = 1e6\n") == Ok(expected) -} - -pub fn parse_float_exponent_uppercase_test() { - let expected = dict.from_list([#("a", tom.Float(1.0e6))]) - assert tom.parse("a = 1E6\n") == Ok(expected) -} - -pub fn parse_float_exponent_postive_test() { - let expected = dict.from_list([#("a", tom.Float(5.0e22))]) - assert tom.parse("a = 5e+22\n") == Ok(expected) -} - -pub fn parse_float_exponent_negative_test() { - let expected = dict.from_list([#("a", tom.Float(-2.0e-22))]) - assert tom.parse("a = -2e-22\n") == Ok(expected) -} - -pub fn parse_float_decimal_and_exponent_test() { - let expected = dict.from_list([#("a", tom.Float(6.626e25))]) - assert tom.parse("a = 6.626e25\n") == Ok(expected) -} - -pub fn parse_float_decimal_and_exponent_positive_test() { - let expected = dict.from_list([#("a", tom.Float(6.626e25))]) - assert tom.parse("a = 6.626e+25\n") == Ok(expected) -} - -pub fn parse_float_decimal_and_exponent_negative_test() { - let expected = dict.from_list([#("a", tom.Float(6.626e-25))]) - assert tom.parse("a = 6.626e-25\n") == Ok(expected) -} - -pub fn parse_date_test() { - let expected = - dict.from_list([#("a", tom.Date(calendar.Date(1979, calendar.May, 27)))]) - assert tom.parse("a = 1979-05-27\n") == Ok(expected) -} - -pub fn parse_time_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 1, 0)))]) - assert tom.parse("a = 07:32:01\n") == Ok(expected) -} - -pub fn parse_time_zero_minute_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 0, 1, 0)))]) - assert tom.parse("a = 07:00:01\n") == Ok(expected) -} - -pub fn parse_time_nanoseconds_999999_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 1, 999_999_000)))]) - assert tom.parse("a = 07:32:01.999999\n") == Ok(expected) -} - -pub fn parse_time_nanoseconds_09179_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 1, 91_790_000)))]) - assert tom.parse("a = 07:32:01.09179\n") == Ok(expected) -} - -pub fn parse_time_nanoseconds_123456789_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 1, 123_456_789)))]) - assert tom.parse("a = 07:32:01.123456789\n") == Ok(expected) -} - -pub fn parse_time_nanoseconds_1_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 1, 100_000_000)))]) - assert tom.parse("a = 07:32:01.1\n") == Ok(expected) -} - -pub fn parse_time_nanoseconds_001_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 1, 1_000_000)))]) - assert tom.parse("a = 07:32:01.001\n") == Ok(expected) -} - -pub fn parse_time_nanoseconds_000000789_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 1, 789)))]) - assert tom.parse("a = 07:32:01.000000789\n") == Ok(expected) -} - -pub fn parse_time_no_seconds_test() { - let expected = - dict.from_list([#("a", tom.Time(calendar.TimeOfDay(7, 32, 0, 0)))]) - assert tom.parse("a = 07:32\n") == Ok(expected) -} - -pub fn parse_date_time_test() { - let expected = - dict.from_list([ - #( - "a", - tom.DateTime( - calendar.Date(1979, calendar.May, 27), - calendar.TimeOfDay(7, 32, 0, 0), - offset: tom.Local, - ), - ), - ]) - assert tom.parse("a = 1979-05-27T07:32:00\n") == Ok(expected) -} - -pub fn parse_date_time_space_test() { - let expected = - dict.from_list([ - #( - "a", - tom.DateTime( - calendar.Date(1979, calendar.May, 27), - calendar.TimeOfDay(7, 0, 1, 0), - offset: tom.Local, - ), - ), - ]) - assert tom.parse("a = 1979-05-27 07:00:01\n") == Ok(expected) -} - -pub fn parse_offset_z_date_time_test() { - let expected = - dict.from_list([ - #( - "a", - tom.DateTime( - calendar.Date(1979, calendar.May, 27), - calendar.TimeOfDay(7, 32, 0, 0), - offset: tom.Offset(calendar.utc_offset), - ), - ), - ]) - assert tom.parse("a = 1979-05-27T07:32:00Z\n") == Ok(expected) -} - -pub fn parse_offset_z_date_time_space_test() { - let expected = - dict.from_list([ - #( - "a", - tom.DateTime( - calendar.Date(1979, calendar.May, 27), - calendar.TimeOfDay(7, 0, 1, 0), - offset: tom.Offset(calendar.utc_offset), - ), - ), - ]) - assert tom.parse("a = 1979-05-27 07:00:01Z\n") == Ok(expected) -} - -pub fn parse_offset_positive_date_time_space_test() { - let expected = - dict.from_list([ - #( - "a", - tom.DateTime( - calendar.Date(1979, calendar.May, 27), - calendar.TimeOfDay(7, 0, 1, 0), - offset: tom.Offset(duration.add( - duration.hours(7), - duration.minutes(40), - )), - ), - ), - ]) - assert tom.parse("a = 1979-05-27 07:00:01+07:40\n") == Ok(expected) -} - -pub fn parse_offset_negative_date_time_space_test() { - let expected = - dict.from_list([ - #( - "a", - tom.DateTime( - calendar.Date(1979, calendar.May, 27), - calendar.TimeOfDay(7, 0, 1, 0), - offset: tom.Offset(duration.add( - duration.hours(-7), - duration.minutes(-1), - )), - ), - ), - ]) - assert tom.parse("a = 1979-05-27 07:00:01-07:01\n") == Ok(expected) -} - -pub fn parse_no_trailing_newline_test() { - let expected = dict.from_list([#("a", tom.Int(1))]) - assert tom.parse("a = 1") == Ok(expected) -} - -pub fn parse_trailing_whitespace_test() { - let expected = dict.from_list([#("a", tom.Int(1))]) - assert tom.parse("a = 1 ") == Ok(expected) -} - -pub fn parse_trailing_other_test() { - assert tom.parse("a = 1 b") == Error(tom.Unexpected("b", "\n")) -} - -pub fn parse_sequence_e_test() { - assert tom.parse("a = \"\\e\"") - == Ok(dict.from_list([#("a", tom.String("\u{001b}"))])) -} - -pub fn parse_sequence_f_test() { - assert tom.parse("a = \"\\f\"") - == Ok(dict.from_list([#("a", tom.String("\f"))])) -} - -pub fn parse_sequence_b_test() { - assert tom.parse("a = \"\\b\"") - == Ok(dict.from_list([#("a", tom.String("\u{0008}"))])) -} - -pub fn parse_ignore_comments_test() { - let expected = dict.from_list([#("field", tom.String("#"))]) - assert tom.parse( - "# This should be ignored -field = \"#\"", - ) - == Ok(expected) -} - -pub fn parse_not_remove_hash_in_string_test() { - let content = tom.Table(dict.from_list([#("field", tom.String("#"))])) - let expected = dict.from_list([#("section", content)]) - assert tom.parse( - "[section] -field = \"#\"", - ) - == Ok(expected) -} - -pub fn parse_multi_line_string_newline_test() { - let expected = - dict.from_list([ - #("contents", tom.String("Hello, Joe!\n")), - ]) - assert tom.parse( - "contents = \"\"\" -Hello, Joe! -\"\"\" -", - ) - == Ok(expected) -} - -pub fn parse_single_string_test() { - let expected = - dict.from_list([ - #("contents", tom.String("Hello, Joe!")), - ]) - assert tom.parse( - "contents = 'Hello, Joe!' -", - ) - == Ok(expected) -} - -pub fn parse_multi_line_single_string_newline_test() { - let expected = - dict.from_list([ - #("contents", tom.String("Hello, Joe!\n")), - ]) - assert tom.parse( - "contents = ''' -Hello, Joe! -''' -", - ) - == Ok(expected) -} - -pub fn parse_multi_line_string_with_hash_test() { - let expected = - dict.from_list([ - #( - "contents", - tom.String( - "document.querySelector(\"#app\").textContent = \"hello\";\n", - ), - ), - ]) - assert tom.parse( - "contents = \"\"\" -document.querySelector(\"#app\").textContent = \"hello\"; -\"\"\" -", - ) - == Ok(expected) -} - -pub fn get_data_in_table_and_inline_table_test() { - let toml = - "section = { field = \"data\" } -another_section = { another_field = \"another_data\", int_field = 2 } -[still_a_section] -still_a_field = 1" - |> tom.parse - assert result.is_ok(toml) == True - use toml <- result.map(toml) - - assert tom.get(toml, ["section", "field"]) == Ok(tom.String("data")) - - assert tom.get(toml, ["another_section", "another_field"]) - == Ok(tom.String("another_data")) - - assert tom.get(toml, ["another_section", "int_field"]) == Ok(tom.Int(2)) - - assert tom.get(toml, ["still_a_section", "still_a_field"]) == Ok(tom.Int(1)) -} - -pub fn tom_as_int_test() { - assert tom.as_int(tom.Int(1)) == Ok(1) - - assert tom.as_int(tom.Float(1.5)) == Error(tom.WrongType([], "Int", "Float")) -} - -pub fn tom_as_float_test() { - assert tom.as_float(tom.Float(1.5)) == Ok(1.5) - - assert tom.as_float(tom.Int(1)) == Error(tom.WrongType([], "Float", "Int")) -} - -pub fn tom_as_bool_test() { - assert tom.as_bool(tom.Bool(True)) == Ok(True) - - assert tom.as_bool(tom.Int(1)) == Error(tom.WrongType([], "Bool", "Int")) -} - -pub fn tom_as_string_test() { - assert tom.as_string(tom.String("hello")) == Ok("hello") - - assert tom.as_string(tom.Int(1)) == Error(tom.WrongType([], "String", "Int")) -} - -pub fn tom_as_date_test() { - let date = calendar.Date(2023, calendar.September, 23) - - assert tom.as_date(tom.Date(date)) == Ok(date) - - assert tom.as_date(tom.Int(1)) == Error(tom.WrongType([], "Date", "Int")) -} - -pub fn tom_as_time_of_day_test() { - let time = calendar.TimeOfDay(12, 30, 0, 4_000_000) - - assert tom.as_time_of_day(tom.Time(time)) == Ok(time) - - assert tom.as_time_of_day(tom.Int(1)) - == Error(tom.WrongType([], "Time", "Int")) -} - -pub fn tom_as_calendar_time_test() { - let date = calendar.Date(2023, calendar.September, 23) - let time_of_day = calendar.TimeOfDay(10, 30, 00, 00) - let offset = tom.Local - - assert tom.as_calendar_time(tom.DateTime(date, time_of_day, offset)) - == Ok(#(date, time_of_day, offset)) - - assert tom.as_calendar_time(tom.Int(1)) - == Error(tom.WrongType([], "DateTime", "Int")) -} - -pub fn tom_as_timestamp_test() { - let date = calendar.Date(1970, calendar.January, 1) - let time_of_day = calendar.TimeOfDay(0, 0, 00, 00) - - assert tom.as_timestamp(tom.Int(1)) - == Error(tom.WrongType([], "DateTime with offset", "Int")) - - assert tom.as_timestamp(tom.DateTime(date, time_of_day, tom.Local)) - == Error(tom.WrongType([], "DateTime with offset", "DateTime")) - - assert tom.as_timestamp(tom.DateTime( - date, - time_of_day, - tom.Offset(duration.seconds(0)), - )) - == Ok(timestamp.from_unix_seconds(0)) -} - -pub fn tom_as_array_test() { - let array = [tom.Int(1), tom.Int(2), tom.Int(3)] - - assert tom.as_array(tom.Array(array)) == Ok(array) - - assert tom.as_array(tom.Int(1)) == Error(tom.WrongType([], "Array", "Int")) -} - -pub fn tom_as_table_test() { - let dict = dict.new() - - assert tom.as_table(tom.Table(dict)) == Ok(dict) - - assert tom.as_table(tom.InlineTable(dict)) == Ok(dict) - - assert tom.as_table(tom.Int(1)) == Error(tom.WrongType([], "Table", "Int")) -} - -pub fn tom_as_number_test() { - assert tom.as_number(tom.Int(1)) == Ok(tom.NumberInt(1)) - - assert tom.as_number(tom.Float(1.5)) == Ok(tom.NumberFloat(1.5)) - - assert tom.as_number(tom.Bool(True)) - == Error(tom.WrongType([], "Number", "Bool")) -} - -pub fn get_date_test() { - let assert Ok(parsed) = tom.parse("a.b.c = 1979-05-27") - - assert tom.get_date(parsed, ["a", "b", "c"]) - == Ok(calendar.Date(1979, calendar.May, 27)) - - assert tom.get_time_of_day(parsed, ["a", "b", "c"]) - == Error(tom.WrongType(["a", "b", "c"], "Time", "Date")) -} - -pub fn get_time_of_day_test() { - let assert Ok(parsed) = tom.parse("a.b.c = 07:32:00") - - assert tom.get_time_of_day(parsed, ["a", "b", "c"]) - == Ok(calendar.TimeOfDay(7, 32, 0, 0)) - - assert tom.get_time_of_day(parsed, ["foo"]) == Error(tom.NotFound(["foo"])) -} - -pub fn get_calendar_time_test() { - let assert Ok(parsed) = tom.parse("a.b.c = 1979-05-27T07:32:00Z") - let expected = #( - calendar.Date(1979, calendar.May, 27), - calendar.TimeOfDay(7, 32, 0, 0), - tom.Offset(calendar.utc_offset), - ) - - assert tom.get_calendar_time(parsed, ["a", "b", "c"]) == Ok(expected) -} - -pub fn get_timestamp_test() { - let assert Ok(parsed) = tom.parse("a.b.c = 1970-01-01T00:00:00Z") - assert tom.get_timestamp(parsed, ["a", "b", "c"]) - == Ok(timestamp.from_unix_seconds(0)) - - let assert Ok(parsed) = tom.parse("a.b.c = 1970-01-01T00:00:00") - assert tom.get_timestamp(parsed, ["a", "b", "c"]) - == Error(tom.WrongType(["a", "b", "c"], "DateTime with offset", "DateTime")) - - let assert Ok(parsed) = tom.parse("a = 1") - assert tom.get_timestamp(parsed, ["a", "b", "c"]) - == Error(tom.WrongType(["a"], "Table", "Int")) -} From f468547649903848f283fa14b0181deeaef20abd Mon Sep 17 00:00:00 2001 From: Louis Pilfold Date: Mon, 13 Apr 2026 12:04:59 +0100 Subject: [PATCH 2/7] Most syntax lexed --- src/tom.gleam | 370 +++++++++++++++++++++++++++++++---- test/tom/tokenise_test.gleam | 175 +++++++++++++++-- 2 files changed, 488 insertions(+), 57 deletions(-) diff --git a/src/tom.gleam b/src/tom.gleam index 875b7f5..7f0d5fc 100644 --- a/src/tom.gleam +++ b/src/tom.gleam @@ -1,3 +1,4 @@ +import gleam/int import gleam/list import gleam/option.{type Option, None, Some} import gleam/result @@ -29,15 +30,15 @@ pub type Token { /// `]]` DoubleRightBracketToken /// A double-quote single-line string. - BasicStringToken(String) + BasicStringToken(src: String, value: String) /// A double-quote multi-line string. - MultiLineBasicStringToken(String) + MultiLineBasicStringToken(src: String, value: String) /// A single-quote single-line string. - LiteralStringToken(String) + LiteralStringToken(src: String) /// A single-quote multi-line string. - MultiLineLiteralStringToken(String) + MultiLineLiteralStringToken(src: String, value: String) /// An unquoted key segment e.g. `wibble` - BareKeyToken(String) + BareKeyToken(value: String) /// An int e.g `123` IntToken(src: String, value: Int) /// A float literal e.g. `123.456` @@ -67,9 +68,9 @@ pub type Sign { /// An error that can occur when parsing a TOML document. pub type TomlError { - /// An unexpected character was encountered when parsing the document. - Unexpected(byte_position: Int, got: String, expected: String) - // /// More than one items have the same key in the document. + UnterminatedString(byte_position: Int) + IncompleteFloat(byte_position: Int) + UnknownSequence(byte_position: Int, got: String) // KeyAlreadyInUse(key: List(String)) } @@ -82,11 +83,15 @@ pub fn to_tokens(src: String) -> Result(List(Token), TomlError) { fn new_lexer(src: String) -> Lexer { let src = string.replace(src, "\r\n", "\n") - let splitters = Splitters(next: splitter.new(["\n", " ", "="])) + let splitters = + Splitters( + literal_string: splitter.new(["\n", "'"]), + multiline_literal_string: splitter.new(["\n", "'''"]), + ) Lexer(0, src:, splitters:) } -fn advance(lexer: Lexer, src: String) -> Lexer { +fn step(lexer: Lexer, src: String) -> Lexer { let position = lexer.position + string.byte_size(lexer.src) - string.byte_size(src) Lexer(..lexer, position:, src:) @@ -97,7 +102,7 @@ type Lexer { } type Splitters { - Splitters(next: Splitter) + Splitters(literal_string: Splitter, multiline_literal_string: Splitter) } fn fold_tokens( @@ -105,7 +110,7 @@ fn fold_tokens( output: output, reduce: fn(output, Int, Token) -> Result(output, TomlError), ) -> Result(output, TomlError) { - case next_token(lexer) { + case lex(lexer) { Ok(#(lexer, EndOfFile as token)) -> { reduce(output, lexer.position, token) } @@ -119,42 +124,325 @@ fn fold_tokens( } } -fn next_token(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { +fn lex(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { case lexer.src { "" -> Ok(#(lexer, EndOfFile)) - "[[" <> src -> Ok(#(advance(lexer, src), DoubleLeftBracketToken)) - "]]" <> src -> Ok(#(advance(lexer, src), DoubleRightBracketToken)) - "\n" <> src -> Ok(#(advance(lexer, src), NewlineToken)) - "[" <> src -> Ok(#(advance(lexer, src), LeftBracketToken)) - "]" <> src -> Ok(#(advance(lexer, src), RightBracketToken)) - "{" <> src -> Ok(#(advance(lexer, src), LeftBraceToken)) - "}" <> src -> Ok(#(advance(lexer, src), RightBraceToken)) - "." <> src -> Ok(#(advance(lexer, src), DotToken)) - "=" <> src -> Ok(#(advance(lexer, src), EqualsToken)) - - "true" <> src -> Ok(#(advance(lexer, src), BoolToken(True))) - "false" <> src -> Ok(#(advance(lexer, src), BoolToken(False))) - - "nan" <> src -> Ok(#(advance(lexer, src), NanToken(None))) - "-nan" <> src -> Ok(#(advance(lexer, src), NanToken(Some(Negative)))) - "+nan" <> src -> Ok(#(advance(lexer, src), NanToken(Some(Positive)))) - - "inf" <> src -> Ok(#(advance(lexer, src), InfinityToken(None))) - "-inf" <> src -> Ok(#(advance(lexer, src), InfinityToken(Some(Negative)))) - "+inf" <> src -> Ok(#(advance(lexer, src), InfinityToken(Some(Positive)))) - - "#" <> src -> { - case string.split_once(src, "\n") { - Ok(#(comment, src)) -> { - Ok(#(advance(lexer, src), CommentToken(comment))) - } - Error(_) -> todo + "[[" <> src -> lexed(lexer, src, DoubleLeftBracketToken) + "]]" <> src -> lexed(lexer, src, DoubleRightBracketToken) + "\n" <> src -> lexed(lexer, src, NewlineToken) + "[" <> src -> lexed(lexer, src, LeftBracketToken) + "]" <> src -> lexed(lexer, src, RightBracketToken) + "{" <> src -> lexed(lexer, src, LeftBraceToken) + "}" <> src -> lexed(lexer, src, RightBraceToken) + "." <> src -> lexed(lexer, src, DotToken) + "," <> src -> lexed(lexer, src, CommaToken) + "=" <> src -> lexed(lexer, src, EqualsToken) + + "true" <> src -> lexed(lexer, src, BoolToken(True)) + "false" <> src -> lexed(lexer, src, BoolToken(False)) + + "nan" <> src -> lexed(lexer, src, NanToken(None)) + "-nan" <> src -> lexed(lexer, src, NanToken(Some(Negative))) + "+nan" <> src -> lexed(lexer, src, NanToken(Some(Positive))) + + "inf" <> src -> lexed(lexer, src, InfinityToken(None)) + "-inf" <> src -> lexed(lexer, src, InfinityToken(Some(Negative))) + "+inf" <> src -> lexed(lexer, src, InfinityToken(Some(Positive))) + + " " <> _ | "\t" <> _ -> lex_whitespace(lexer) + "'''" <> src -> lex_multiline_literal_string(step(lexer, src)) + "'" <> src -> lex_literal_string(step(lexer, src)) + "#" <> src -> lex_comment(src, lexer) + + "0" <> src -> lex_number(step(lexer, src), 0, "0") + "1" <> src -> lex_number(step(lexer, src), 1, "1") + "2" <> src -> lex_number(step(lexer, src), 2, "2") + "3" <> src -> lex_number(step(lexer, src), 3, "3") + "4" <> src -> lex_number(step(lexer, src), 4, "4") + "5" <> src -> lex_number(step(lexer, src), 5, "5") + "6" <> src -> lex_number(step(lexer, src), 6, "6") + "7" <> src -> lex_number(step(lexer, src), 7, "7") + "8" <> src -> lex_number(step(lexer, src), 8, "8") + "9" <> src -> lex_number(step(lexer, src), 9, "9") + "+0" <> src -> lex_number(step(lexer, src), 0, "+0") + "+1" <> src -> lex_number(step(lexer, src), 1, "+1") + "+2" <> src -> lex_number(step(lexer, src), 2, "+2") + "+3" <> src -> lex_number(step(lexer, src), 3, "+3") + "+4" <> src -> lex_number(step(lexer, src), 4, "+4") + "+5" <> src -> lex_number(step(lexer, src), 5, "+5") + "+6" <> src -> lex_number(step(lexer, src), 6, "+6") + "+7" <> src -> lex_number(step(lexer, src), 7, "+7") + "+8" <> src -> lex_number(step(lexer, src), 8, "+8") + "+9" <> src -> lex_number(step(lexer, src), 9, "+9") + "-0" <> src -> lex_number(step(lexer, src), 0, "-0") + "-1" <> src -> lex_number(step(lexer, src), 1, "-1") + "-2" <> src -> lex_number(step(lexer, src), 2, "-2") + "-3" <> src -> lex_number(step(lexer, src), 3, "-3") + "-4" <> src -> lex_number(step(lexer, src), 4, "-4") + "-5" <> src -> lex_number(step(lexer, src), 5, "-5") + "-6" <> src -> lex_number(step(lexer, src), 6, "-6") + "-7" <> src -> lex_number(step(lexer, src), 7, "-7") + "-8" <> src -> lex_number(step(lexer, src), 8, "-8") + "-9" <> src -> lex_number(step(lexer, src), 9, "-9") + + _ -> lex_bare_key(lexer, "") + } +} + +fn lex_number( + lexer: Lexer, + int: Int, + text: String, +) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "_" <> src -> lex_number(step(lexer, src), int, text <> "_") + "0" <> src -> lex_number(step(lexer, src), int * 10 + 0, text <> "0") + "1" <> src -> lex_number(step(lexer, src), int * 10 + 1, text <> "1") + "2" <> src -> lex_number(step(lexer, src), int * 10 + 2, text <> "2") + "3" <> src -> lex_number(step(lexer, src), int * 10 + 3, text <> "3") + "4" <> src -> lex_number(step(lexer, src), int * 10 + 4, text <> "4") + "5" <> src -> lex_number(step(lexer, src), int * 10 + 5, text <> "5") + "6" <> src -> lex_number(step(lexer, src), int * 10 + 6, text <> "6") + "7" <> src -> lex_number(step(lexer, src), int * 10 + 7, text <> "7") + "8" <> src -> lex_number(step(lexer, src), int * 10 + 8, text <> "8") + "9" <> src -> lex_number(step(lexer, src), int * 10 + 9, text <> "9") + + "." <> src -> { + let float = int.to_float(int) + lex_float(step(lexer, src), float, 0.1, text <> ".") + } + + src -> { + let value = case text { + "-" <> _ -> -int + _ -> int } + lexed(lexer, src, IntToken(text, value:)) } + } +} + +fn lex_float( + lexer: Lexer, + float: Float, + unit: Float, + text: String, +) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "_" <> src -> lex_float(step(lexer, src), float, unit, text <> "_") + "0" <> src -> lex_float(step(lexer, src), float, unit *. 0.1, text <> "0") + "1" <> src -> { + let float = float +. 1.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "1") + } + "2" <> src -> { + let float = float +. 2.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "2") + } + "3" <> src -> { + let float = float +. 3.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "3") + } + "4" <> src -> { + let float = float +. 4.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "4") + } + "5" <> src -> { + let float = float +. 5.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "5") + } + "6" <> src -> { + let float = float +. 6.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "6") + } + "7" <> src -> { + let float = float +. 7.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "7") + } + "8" <> src -> { + let float = float +. 8.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "8") + } + "9" <> src -> { + let float = float +. 9.0 *. unit + lex_float(step(lexer, src), float, unit *. 0.1, text <> "9") + } + + // ["e", "+", ..src] -> parse_exponent(src, number, sign, 0, Positive) + // ["e", "-", ..src] -> parse_exponent(src, number, sign, 0, Negative) + // ["e", ..src] -> parse_exponent(src, number, sign, 0, Positive) + // ["E", "+", ..src] -> parse_exponent(src, number, sign, 0, Positive) + // ["E", "-", ..src] -> parse_exponent(src, number, sign, 0, Negative) + // ["E", ..src] -> parse_exponent(src, number, sign, 0, Positive) + _ if unit == 0.1 -> Error(IncompleteFloat(lexer.position - 1)) src -> { - todo as src + let value = case text { + "-" <> _ -> -1.0 *. float + _ -> float + } + lexed(lexer, src, FloatToken(text, value:)) } } } + +fn lex_whitespace(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { + let #(whitespace, src) = take_whitespace("", lexer.src) + lexed(lexer, src, WhitespaceToken(whitespace)) +} + +fn lex_bare_key( + lexer: Lexer, + content: String, +) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "A" as c <> src + | "B" as c <> src + | "C" as c <> src + | "D" as c <> src + | "E" as c <> src + | "F" as c <> src + | "G" as c <> src + | "H" as c <> src + | "I" as c <> src + | "J" as c <> src + | "K" as c <> src + | "L" as c <> src + | "M" as c <> src + | "N" as c <> src + | "O" as c <> src + | "P" as c <> src + | "Q" as c <> src + | "R" as c <> src + | "S" as c <> src + | "T" as c <> src + | "U" as c <> src + | "V" as c <> src + | "W" as c <> src + | "X" as c <> src + | "Y" as c <> src + | "Z" as c <> src + | "a" as c <> src + | "b" as c <> src + | "c" as c <> src + | "d" as c <> src + | "e" as c <> src + | "f" as c <> src + | "g" as c <> src + | "h" as c <> src + | "i" as c <> src + | "j" as c <> src + | "k" as c <> src + | "l" as c <> src + | "m" as c <> src + | "n" as c <> src + | "o" as c <> src + | "p" as c <> src + | "q" as c <> src + | "r" as c <> src + | "s" as c <> src + | "t" as c <> src + | "u" as c <> src + | "v" as c <> src + | "w" as c <> src + | "x" as c <> src + | "y" as c <> src + | "z" as c <> src + | "0" as c <> src + | "1" as c <> src + | "2" as c <> src + | "3" as c <> src + | "4" as c <> src + | "5" as c <> src + | "6" as c <> src + | "7" as c <> src + | "8" as c <> src + | "9" as c <> src + | "-" as c <> src + | "_" as c <> src -> lex_bare_key(step(lexer, src), content <> c) + + src if content != "" -> lexed(lexer, src, BareKeyToken(content)) + src -> { + let got = case string.pop_grapheme(src) { + Ok(#(got, _)) -> got + Error(_) -> "" + } + Error(UnknownSequence(lexer.position, got)) + } + } +} + +fn lex_comment(src: String, lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { + case string.split_once(src, "\n") { + Ok(#(comment, src)) -> { + let token = CommentToken(comment <> "\n") + lexed(lexer, src, token) + } + + Error(_) -> { + let token = CommentToken(src) + lexed(lexer, "", token) + } + } +} + +fn lex_multiline_literal_string( + lexer: Lexer, +) -> Result(#(Lexer, Token), TomlError) { + case string.split_once(lexer.src, "'''") { + Ok(#(content, src)) -> { + let value = drop_leading_newline(content) + lexed(lexer, src, MultiLineLiteralStringToken(content, value:)) + } + Error(_) -> { + Error(UnterminatedString(lexer.position - 3)) + } + } +} + +fn drop_leading_newline(src: String) -> String { + case src { + "\n" <> src -> src + src -> src + } +} + +fn lex_literal_string(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { + let start_position = lexer.position - 1 + let #(lexer, before, split) = + run_splitter(lexer, lexer.splitters.literal_string, lexer.src) + case split { + "'" -> Ok(#(lexer, LiteralStringToken(before))) + "\n" -> Error(UnterminatedString(start_position)) + _ -> Error(UnterminatedString(start_position)) + } +} + +fn run_splitter( + lexer: Lexer, + splitter: Splitter, + src: String, +) -> #(Lexer, String, String) { + let #(before, split, after) = splitter.split(splitter, src) + let lexer = step(lexer, after) + #(lexer, before, split) +} + +fn lexed( + lexer: Lexer, + src: String, + token: Token, +) -> Result(#(Lexer, Token), TomlError) { + Ok(#(step(lexer, src), token)) +} + +fn take_whitespace(taken: String, src: String) -> #(String, String) { + case src { + // Process indentation in 4 space batches, to reduce loop iterations. + " " <> src -> take_whitespace(taken <> " ", src) + " " <> src -> take_whitespace(taken <> " ", src) + "\t" <> src -> take_whitespace(taken <> "\t", src) + _ -> #(taken, src) + } +} diff --git a/test/tom/tokenise_test.gleam b/test/tom/tokenise_test.gleam index 8645247..7caebe2 100644 --- a/test/tom/tokenise_test.gleam +++ b/test/tom/tokenise_test.gleam @@ -1,9 +1,13 @@ import gleam/option.{None, Some} import tom.{ - BareKeyToken, BoolToken, CommentToken, DotToken, DoubleLeftBracketToken, - DoubleRightBracketToken, EndOfFile, EqualsToken, InfinityToken, IntToken, - LeftBraceToken, LeftBracketToken, NanToken, Negative, NewlineToken, Positive, - RightBraceToken, RightBracketToken, WhitespaceToken, + BareKeyToken, BasicStringToken, BoolToken, CommaToken, CommentToken, DotToken, + DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFile, EqualsToken, + FloatToken, IncompleteFloat, InfinityToken, IntToken, LeftBraceToken, + LeftBracketToken, LiteralStringToken, LocalDateTimeToken, LocalDateToken, + LocalTimeToken, MultiLineBasicStringToken, MultiLineLiteralStringToken, + NanToken, Negative, NewlineToken, OffsetDateTimeToken, Positive, + RightBraceToken, RightBracketToken, UnknownSequence, UnterminatedString, + WhitespaceToken, } pub fn empty_test() { @@ -18,6 +22,10 @@ pub fn dot_test() { assert tom.to_tokens(".") == Ok([DotToken, EndOfFile]) } +pub fn comma_test() { + assert tom.to_tokens(",") == Ok([CommaToken, EndOfFile]) +} + pub fn braces_test() { assert tom.to_tokens("{}") == Ok([LeftBraceToken, RightBraceToken, EndOfFile]) } @@ -71,15 +79,150 @@ pub fn newline_test() { pub fn comment_test() { assert tom.to_tokens("# Hello, world!\n\n") - == Ok([CommentToken(" Hello, world!"), NewlineToken, EndOfFile]) -} -// pub fn key_value_with_whitespace_tokenises_test() { -// assert tom2.to_tokens("answer = 42") -// == Ok([ -// BareKeyToken("answer"), -// WhitespaceToken(" "), -// EqualsToken, -// WhitespaceToken(" "), -// IntToken("42", 42), -// ]) -// } + == Ok([CommentToken(" Hello, world!\n"), NewlineToken, EndOfFile]) +} + +pub fn comment_no_newline_test() { + assert tom.to_tokens("# Hello, world!") + == Ok([CommentToken(" Hello, world!"), EndOfFile]) +} + +pub fn spaces_test() { + assert tom.to_tokens( + "# 1 + # 2 +", + ) + == Ok([ + CommentToken(" 1\n"), + WhitespaceToken(" "), + CommentToken(" 2\n"), + EndOfFile, + ]) +} + +pub fn tabs_test() { + assert tom.to_tokens( + "# 1 +\t\t\t# 2 +", + ) + == Ok([ + CommentToken(" 1\n"), + WhitespaceToken("\t\t\t"), + CommentToken(" 2\n"), + EndOfFile, + ]) +} + +pub fn tabs_and_spaces_test() { + assert tom.to_tokens( + "# 1 +\t \t \t# 2 +", + ) + == Ok([ + CommentToken(" 1\n"), + WhitespaceToken("\t \t \t"), + CommentToken(" 2\n"), + EndOfFile, + ]) +} + +pub fn literal_string_test() { + assert tom.to_tokens("'Hello'") + == Ok([ + LiteralStringToken(src: "Hello"), + EndOfFile, + ]) +} + +pub fn literal_string_newline_test() { + assert tom.to_tokens("'1\n2'") == Error(UnterminatedString(byte_position: 0)) +} + +pub fn literal_string_unterminated_test() { + assert tom.to_tokens("'1") == Error(UnterminatedString(byte_position: 0)) +} + +pub fn multiline_literal_string_test() { + assert tom.to_tokens( + "''' +1 +2 +3 +'''", + ) + == Ok([ + MultiLineLiteralStringToken(src: "\n1\n2\n3\n", value: "1\n2\n3\n"), + EndOfFile, + ]) +} + +pub fn unexpected_test() { + assert tom.to_tokens("???") == Error(UnknownSequence(0, "?")) +} + +pub fn key_test() { + assert tom.to_tokens("name") == Ok([BareKeyToken("name"), EndOfFile]) +} + +pub fn key_fancy_test() { + assert tom.to_tokens("_H311o-W0rld_") + == Ok([BareKeyToken("_H311o-W0rld_"), EndOfFile]) +} + +pub fn key_value_test() { + assert tom.to_tokens("items = []") + == Ok([ + BareKeyToken("items"), + WhitespaceToken(" "), + EqualsToken, + WhitespaceToken(" "), + LeftBracketToken, + RightBracketToken, + EndOfFile, + ]) +} + +pub fn number_test() { + assert tom.to_tokens("1234567890") + == Ok([IntToken("1234567890", 1_234_567_890), EndOfFile]) +} + +pub fn int_positive_test() { + assert tom.to_tokens("+1234567890") + == Ok([IntToken("+1234567890", 1_234_567_890), EndOfFile]) +} + +pub fn int_negative_test() { + assert tom.to_tokens("-1234567890") + == Ok([IntToken("-1234567890", -1_234_567_890), EndOfFile]) +} + +pub fn int_underscore_test() { + assert tom.to_tokens("12_345_67__890") + == Ok([IntToken("12_345_67__890", 1_234_567_890), EndOfFile]) +} + +pub fn float_test() { + assert tom.to_tokens("12.34") == Ok([FloatToken("12.34", 12.34), EndOfFile]) +} + +pub fn float_negative_test() { + assert tom.to_tokens("-12.34") + == Ok([FloatToken("-12.34", -12.34), EndOfFile]) +} + +pub fn float_positive_test() { + assert tom.to_tokens("+12.34") == Ok([FloatToken("+12.34", 12.34), EndOfFile]) +} + +pub fn float_underscore_test() { + assert tom.to_tokens("12_34.5_67__890") + == Ok([FloatToken("12_34.5_67__890", 1234.567_89), EndOfFile]) +} + +pub fn float_incomplete_test() { + assert tom.to_tokens("12.") == Error(IncompleteFloat(2)) +} From 40e85496336b0781026dad4031e66438caf9364d Mon Sep 17 00:00:00 2001 From: Louis Pilfold Date: Mon, 13 Apr 2026 12:22:36 +0100 Subject: [PATCH 3/7] Lex scientific notation --- src/tom.gleam | 118 +++++++++++++++++++++++++++++++++-- test/tom/tokenise_test.gleam | 28 +++++++++ 2 files changed, 140 insertions(+), 6 deletions(-) diff --git a/src/tom.gleam b/src/tom.gleam index 7f0d5fc..c2d8682 100644 --- a/src/tom.gleam +++ b/src/tom.gleam @@ -1,3 +1,4 @@ +import gleam/float import gleam/int import gleam/list import gleam/option.{type Option, None, Some} @@ -213,6 +214,31 @@ fn lex_number( lex_float(step(lexer, src), float, 0.1, text <> ".") } + "e+" <> src -> { + let float = int.to_float(int) + lex_exponent(step(lexer, src), float, text <> "e+", 0, Positive) + } + "e-" <> src -> { + let float = int.to_float(int) + lex_exponent(step(lexer, src), float, text <> "e-", 0, Negative) + } + "e" <> src -> { + let float = int.to_float(int) + lex_exponent(step(lexer, src), float, text <> "e", 0, Positive) + } + "E+" <> src -> { + let float = int.to_float(int) + lex_exponent(step(lexer, src), float, text <> "E+", 0, Positive) + } + "E-" <> src -> { + let float = int.to_float(int) + lex_exponent(step(lexer, src), float, text <> "E-", 0, Negative) + } + "E" <> src -> { + let float = int.to_float(int) + lex_exponent(step(lexer, src), float, text <> "E", 0, Positive) + } + src -> { let value = case text { "-" <> _ -> -int @@ -269,12 +295,19 @@ fn lex_float( lex_float(step(lexer, src), float, unit *. 0.1, text <> "9") } - // ["e", "+", ..src] -> parse_exponent(src, number, sign, 0, Positive) - // ["e", "-", ..src] -> parse_exponent(src, number, sign, 0, Negative) - // ["e", ..src] -> parse_exponent(src, number, sign, 0, Positive) - // ["E", "+", ..src] -> parse_exponent(src, number, sign, 0, Positive) - // ["E", "-", ..src] -> parse_exponent(src, number, sign, 0, Negative) - // ["E", ..src] -> parse_exponent(src, number, sign, 0, Positive) + "e+" <> src -> + lex_exponent(step(lexer, src), float, text <> "e+", 0, Positive) + "e-" <> src -> + lex_exponent(step(lexer, src), float, text <> "e-", 0, Negative) + "e" <> src -> + lex_exponent(step(lexer, src), float, text <> "e", 0, Positive) + "E+" <> src -> + lex_exponent(step(lexer, src), float, text <> "E+", 0, Positive) + "E-" <> src -> + lex_exponent(step(lexer, src), float, text <> "E-", 0, Negative) + "E" <> src -> + lex_exponent(step(lexer, src), float, text <> "E", 0, Positive) + _ if unit == 0.1 -> Error(IncompleteFloat(lexer.position - 1)) src -> { @@ -287,6 +320,79 @@ fn lex_float( } } +fn lex_exponent( + lexer: Lexer, + n: Float, + text: String, + ex: Int, + sign: Sign, +) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "_" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "_", ex, sign) + } + "0" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "0", ex * 10, sign) + } + "1" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "1", ex * 10 + 1, sign) + } + "2" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "2", ex * 10 + 2, sign) + } + "3" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "3", ex * 10 + 3, sign) + } + "4" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "4", ex * 10 + 4, sign) + } + "5" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "5", ex * 10 + 5, sign) + } + "6" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "6", ex * 10 + 6, sign) + } + "7" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "7", ex * 10 + 7, sign) + } + "8" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "8", ex * 10 + 8, sign) + } + "9" <> src -> { + let lexer = step(lexer, src) + lex_exponent(lexer, n, text <> "9", ex * 10 + 9, sign) + } + + // Anything else and the number is terminated + src -> { + let n = case text { + "-" <> _ -> n *. -1.0 + _ -> n + } + let exponent = + int.to_float(case sign { + Positive -> ex + Negative -> -ex + }) + let multiplier = case float.power(10.0, exponent) { + Ok(multiplier) -> multiplier + Error(_) -> 1.0 + } + lexed(lexer, src, FloatToken(text, n *. multiplier)) + } + } +} + fn lex_whitespace(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { let #(whitespace, src) = take_whitespace("", lexer.src) lexed(lexer, src, WhitespaceToken(whitespace)) diff --git a/test/tom/tokenise_test.gleam b/test/tom/tokenise_test.gleam index 7caebe2..8fc483d 100644 --- a/test/tom/tokenise_test.gleam +++ b/test/tom/tokenise_test.gleam @@ -226,3 +226,31 @@ pub fn float_underscore_test() { pub fn float_incomplete_test() { assert tom.to_tokens("12.") == Error(IncompleteFloat(2)) } + +pub fn lex_float_exponent_test() { + assert tom.to_tokens("1e6") == Ok([FloatToken("1e6", 1.0e6), EndOfFile]) +} + +pub fn lex_float_exponent_uppercase_test() { + assert tom.to_tokens("1E6") == Ok([FloatToken("1E6", 1.0e6), EndOfFile]) +} + +pub fn lex_float_exponent_negative_test() { + assert tom.to_tokens("-2e-22") + == Ok([FloatToken("-2e-22", -2.0e-22), EndOfFile]) +} + +pub fn lex_float_decimal_and_exponent_test() { + assert tom.to_tokens("6.626e25") + == Ok([FloatToken("6.626e25", 6.626e25), EndOfFile]) +} + +pub fn lex_float_decimal_and_exponent_positive_test() { + assert tom.to_tokens("6.626e+25") + == Ok([FloatToken("6.626e+25", 6.626e25), EndOfFile]) +} + +pub fn lex_float_decimal_and_exponent_negative_test() { + assert tom.to_tokens("6.626e-25") + == Ok([FloatToken("6.626e-25", 6.626e-25), EndOfFile]) +} From 3d27862690c1a79299bb6e8ca64fe7469c494489 Mon Sep 17 00:00:00 2001 From: Louis Pilfold Date: Mon, 13 Apr 2026 13:42:07 +0100 Subject: [PATCH 4/7] Times --- src/tom.gleam | 191 ++++++++++++++++++++++++++++++++++- test/tom/tokenise_test.gleam | 45 +++++++-- 2 files changed, 225 insertions(+), 11 deletions(-) diff --git a/src/tom.gleam b/src/tom.gleam index c2d8682..929fe12 100644 --- a/src/tom.gleam +++ b/src/tom.gleam @@ -4,6 +4,8 @@ import gleam/list import gleam/option.{type Option, None, Some} import gleam/result import gleam/string +import gleam/time/calendar +import gleam/time/duration import splitter.{type Splitter} /// A token produced by lexing TOML source text. @@ -51,13 +53,18 @@ pub type Token { /// `true` or `false`. BoolToken(value: Bool) /// A date-time with an offset. - OffsetDateTimeToken(String) + OffsetDateTimeToken( + src: String, + date: calendar.Date, + time: calendar.TimeOfDay, + offset: duration.Duration, + ) /// A date-time with no offset. - LocalDateTimeToken(String) + LocalDateTimeToken(src: String, date: calendar.Date, time: calendar.TimeOfDay) /// A date. - LocalDateToken(String) + LocalDateToken(src: String, date: calendar.Date) /// A time. - LocalTimeToken(String) + LocalTimeToken(src: String, time: calendar.TimeOfDay) /// The end! EndOfFile } @@ -71,6 +78,8 @@ pub type Sign { pub type TomlError { UnterminatedString(byte_position: Int) IncompleteFloat(byte_position: Int) + IncompleteDate(byte_position: Int) + IncompleteTime(byte_position: Int) UnknownSequence(byte_position: Int, got: String) // KeyAlreadyInUse(key: List(String)) } @@ -88,6 +97,8 @@ fn new_lexer(src: String) -> Lexer { Splitters( literal_string: splitter.new(["\n", "'"]), multiline_literal_string: splitter.new(["\n", "'''"]), + basic_string: splitter.new(["\n", "\"", "\\\\", "\\"]), + multiline_basic_string: splitter.new(["\n", "\"\"\"", "\\\\", "\\"]), ) Lexer(0, src:, splitters:) } @@ -103,7 +114,12 @@ type Lexer { } type Splitters { - Splitters(literal_string: Splitter, multiline_literal_string: Splitter) + Splitters( + literal_string: Splitter, + multiline_literal_string: Splitter, + basic_string: Splitter, + multiline_basic_string: Splitter, + ) } fn fold_tokens( @@ -209,6 +225,9 @@ fn lex_number( "8" <> src -> lex_number(step(lexer, src), int * 10 + 8, text <> "8") "9" <> src -> lex_number(step(lexer, src), int * 10 + 9, text <> "9") + ":" <> src if int < 24 -> + lex_time_minute(step(lexer, src), int, text <> ":") + "." <> src -> { let float = int.to_float(int) lex_float(step(lexer, src), float, 0.1, text <> ".") @@ -249,6 +268,168 @@ fn lex_number( } } +fn lex_time_minute( + lexer: Lexer, + hours: Int, + text: String, +) -> Result(#(Lexer, Token), TomlError) { + use #(lexer, text, minutes) <- result.try(lex_number_under_60(lexer, text)) + use #(lexer, text, seconds, ns) <- result.try(lex_seconds(lexer, text)) + let time = calendar.TimeOfDay(hours, minutes, seconds, ns) + Ok(#(lexer, LocalTimeToken(text, time))) +} + +fn lex_seconds( + lexer: Lexer, + text: String, +) -> Result(#(Lexer, String, Int, Int), TomlError) { + case lexer.src { + ":" <> src -> { + let text = text <> ":" + let lexer = step(lexer, src) + use #(lexer, text, seconds) <- result.try(lex_number_under_60(lexer, text)) + case lexer.src { + "." <> src -> { + let lexer = step(lexer, src) + let text = text <> "." + parse_time_ns(lexer, text, seconds, 0, 0) + } + _ -> Ok(#(lexer, text, seconds, 0)) + } + } + + _ -> Ok(#(lexer, text, 0, 0)) + } +} + +fn parse_time_ns( + lexer: Lexer, + text: String, + seconds: Int, + ns: Int, + digits_count: Int, +) -> Result(#(Lexer, String, Int, Int), TomlError) { + case lexer.src { + "0" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "0", seconds, ns * 10 + 0, digits_count + 1) + } + "1" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "1", seconds, ns * 10 + 1, digits_count + 1) + } + "2" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "2", seconds, ns * 10 + 2, digits_count + 1) + } + "3" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "3", seconds, ns * 10 + 3, digits_count + 1) + } + "4" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "4", seconds, ns * 10 + 4, digits_count + 1) + } + "5" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "5", seconds, ns * 10 + 5, digits_count + 1) + } + "6" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "6", seconds, ns * 10 + 6, digits_count + 1) + } + "7" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "7", seconds, ns * 10 + 7, digits_count + 1) + } + "8" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "8", seconds, ns * 10 + 8, digits_count + 1) + } + "9" <> src if digits_count < 9 -> { + let lexer = step(lexer, src) + parse_time_ns(lexer, text <> "9", seconds, ns * 10 + 9, digits_count + 1) + } + + _ if digits_count == 0 -> Error(IncompleteTime(lexer.position)) + + _ -> { + let exponent = int.to_float(9 - digits_count) + let assert Ok(multiplier) = float.power(10.0, exponent) + Ok(#(lexer, text, seconds, ns * float.truncate(multiplier))) + } + } +} + +fn lex_number_under_60( + lexer: Lexer, + text: String, +) -> Result(#(Lexer, String, Int), TomlError) { + case lexer.src { + "00" <> src -> Ok(#(step(lexer, src), text <> "00", 0)) + "01" <> src -> Ok(#(step(lexer, src), text <> "01", 1)) + "02" <> src -> Ok(#(step(lexer, src), text <> "02", 2)) + "03" <> src -> Ok(#(step(lexer, src), text <> "03", 3)) + "04" <> src -> Ok(#(step(lexer, src), text <> "04", 4)) + "05" <> src -> Ok(#(step(lexer, src), text <> "05", 5)) + "06" <> src -> Ok(#(step(lexer, src), text <> "06", 6)) + "07" <> src -> Ok(#(step(lexer, src), text <> "07", 7)) + "08" <> src -> Ok(#(step(lexer, src), text <> "08", 8)) + "09" <> src -> Ok(#(step(lexer, src), text <> "09", 9)) + "10" <> src -> Ok(#(step(lexer, src), text <> "10", 10)) + "11" <> src -> Ok(#(step(lexer, src), text <> "11", 11)) + "12" <> src -> Ok(#(step(lexer, src), text <> "12", 12)) + "13" <> src -> Ok(#(step(lexer, src), text <> "13", 13)) + "14" <> src -> Ok(#(step(lexer, src), text <> "14", 14)) + "15" <> src -> Ok(#(step(lexer, src), text <> "15", 15)) + "16" <> src -> Ok(#(step(lexer, src), text <> "16", 16)) + "17" <> src -> Ok(#(step(lexer, src), text <> "17", 17)) + "18" <> src -> Ok(#(step(lexer, src), text <> "18", 18)) + "19" <> src -> Ok(#(step(lexer, src), text <> "19", 19)) + "20" <> src -> Ok(#(step(lexer, src), text <> "20", 20)) + "21" <> src -> Ok(#(step(lexer, src), text <> "21", 21)) + "22" <> src -> Ok(#(step(lexer, src), text <> "22", 22)) + "23" <> src -> Ok(#(step(lexer, src), text <> "23", 23)) + "24" <> src -> Ok(#(step(lexer, src), text <> "24", 24)) + "25" <> src -> Ok(#(step(lexer, src), text <> "25", 25)) + "26" <> src -> Ok(#(step(lexer, src), text <> "26", 26)) + "27" <> src -> Ok(#(step(lexer, src), text <> "27", 27)) + "28" <> src -> Ok(#(step(lexer, src), text <> "28", 28)) + "29" <> src -> Ok(#(step(lexer, src), text <> "29", 29)) + "30" <> src -> Ok(#(step(lexer, src), text <> "30", 30)) + "31" <> src -> Ok(#(step(lexer, src), text <> "31", 31)) + "32" <> src -> Ok(#(step(lexer, src), text <> "32", 32)) + "33" <> src -> Ok(#(step(lexer, src), text <> "33", 33)) + "34" <> src -> Ok(#(step(lexer, src), text <> "34", 34)) + "35" <> src -> Ok(#(step(lexer, src), text <> "35", 35)) + "36" <> src -> Ok(#(step(lexer, src), text <> "36", 36)) + "37" <> src -> Ok(#(step(lexer, src), text <> "37", 37)) + "38" <> src -> Ok(#(step(lexer, src), text <> "38", 38)) + "39" <> src -> Ok(#(step(lexer, src), text <> "39", 39)) + "40" <> src -> Ok(#(step(lexer, src), text <> "40", 40)) + "41" <> src -> Ok(#(step(lexer, src), text <> "41", 41)) + "42" <> src -> Ok(#(step(lexer, src), text <> "42", 42)) + "43" <> src -> Ok(#(step(lexer, src), text <> "43", 43)) + "44" <> src -> Ok(#(step(lexer, src), text <> "44", 44)) + "45" <> src -> Ok(#(step(lexer, src), text <> "45", 45)) + "46" <> src -> Ok(#(step(lexer, src), text <> "46", 46)) + "47" <> src -> Ok(#(step(lexer, src), text <> "47", 47)) + "48" <> src -> Ok(#(step(lexer, src), text <> "48", 48)) + "49" <> src -> Ok(#(step(lexer, src), text <> "49", 49)) + "50" <> src -> Ok(#(step(lexer, src), text <> "50", 50)) + "51" <> src -> Ok(#(step(lexer, src), text <> "51", 51)) + "52" <> src -> Ok(#(step(lexer, src), text <> "52", 52)) + "53" <> src -> Ok(#(step(lexer, src), text <> "53", 53)) + "54" <> src -> Ok(#(step(lexer, src), text <> "54", 54)) + "55" <> src -> Ok(#(step(lexer, src), text <> "55", 55)) + "56" <> src -> Ok(#(step(lexer, src), text <> "56", 56)) + "57" <> src -> Ok(#(step(lexer, src), text <> "57", 57)) + "58" <> src -> Ok(#(step(lexer, src), text <> "58", 58)) + "59" <> src -> Ok(#(step(lexer, src), text <> "59", 59)) + _ -> Error(IncompleteTime(lexer.position)) + } +} + fn lex_float( lexer: Lexer, float: Float, diff --git a/test/tom/tokenise_test.gleam b/test/tom/tokenise_test.gleam index 8fc483d..4257048 100644 --- a/test/tom/tokenise_test.gleam +++ b/test/tom/tokenise_test.gleam @@ -1,13 +1,14 @@ import gleam/option.{None, Some} +import gleam/time/calendar import tom.{ BareKeyToken, BasicStringToken, BoolToken, CommaToken, CommentToken, DotToken, DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFile, EqualsToken, - FloatToken, IncompleteFloat, InfinityToken, IntToken, LeftBraceToken, - LeftBracketToken, LiteralStringToken, LocalDateTimeToken, LocalDateToken, - LocalTimeToken, MultiLineBasicStringToken, MultiLineLiteralStringToken, - NanToken, Negative, NewlineToken, OffsetDateTimeToken, Positive, - RightBraceToken, RightBracketToken, UnknownSequence, UnterminatedString, - WhitespaceToken, + FloatToken, IncompleteFloat, IncompleteTime, InfinityToken, IntToken, + LeftBraceToken, LeftBracketToken, LiteralStringToken, LocalDateTimeToken, + LocalDateToken, LocalTimeToken, MultiLineBasicStringToken, + MultiLineLiteralStringToken, NanToken, Negative, NewlineToken, + OffsetDateTimeToken, Positive, RightBraceToken, RightBracketToken, + UnknownSequence, UnterminatedString, WhitespaceToken, } pub fn empty_test() { @@ -254,3 +255,35 @@ pub fn lex_float_decimal_and_exponent_negative_test() { assert tom.to_tokens("6.626e-25") == Ok([FloatToken("6.626e-25", 6.626e-25), EndOfFile]) } + +pub fn local_time_test() { + assert tom.to_tokens("07:32:00") + == Ok([ + LocalTimeToken("07:32:00", calendar.TimeOfDay(7, 32, 0, 0)), + EndOfFile, + ]) +} + +pub fn local_time_fractional_seconds_test() { + assert tom.to_tokens("00:32:00.1234") + == Ok([ + LocalTimeToken("00:32:00.1234", calendar.TimeOfDay(0, 32, 0, 123_400_000)), + EndOfFile, + ]) +} + +pub fn local_time_incomplete_minutes_test() { + assert tom.to_tokens("07:") == Error(IncompleteTime(3)) +} + +pub fn local_time_incomplete_seconds_test() { + assert tom.to_tokens("07:32") + == Ok([ + LocalTimeToken("07:32", calendar.TimeOfDay(7, 32, 0, 0)), + EndOfFile, + ]) +} + +pub fn local_time_incomplete_fractional_seconds_test() { + assert tom.to_tokens("07:32:00.") == Error(IncompleteTime(9)) +} From 5fbc85cdd1e39776ebab5e0382618c3de024bb5b Mon Sep 17 00:00:00 2001 From: Louis Pilfold Date: Mon, 13 Apr 2026 15:05:37 +0100 Subject: [PATCH 5/7] String lexing --- src/tom.gleam | 245 ++++++++++++++++++++++++++++++++++- test/tom/tokenise_test.gleam | 179 ++++++++++++++++++++++++- 2 files changed, 414 insertions(+), 10 deletions(-) diff --git a/src/tom.gleam b/src/tom.gleam index 929fe12..ae4f16d 100644 --- a/src/tom.gleam +++ b/src/tom.gleam @@ -4,8 +4,11 @@ import gleam/list import gleam/option.{type Option, None, Some} import gleam/result import gleam/string -import gleam/time/calendar -import gleam/time/duration +import gleam/time/calendar.{ + type Month, April, August, December, February, January, July, June, March, May, + November, October, September, +} +import gleam/time/duration.{type Duration} import splitter.{type Splitter} /// A token produced by lexing TOML source text. @@ -57,7 +60,7 @@ pub type Token { src: String, date: calendar.Date, time: calendar.TimeOfDay, - offset: duration.Duration, + offset: Duration, ) /// A date-time with no offset. LocalDateTimeToken(src: String, date: calendar.Date, time: calendar.TimeOfDay) @@ -80,7 +83,8 @@ pub type TomlError { IncompleteFloat(byte_position: Int) IncompleteDate(byte_position: Int) IncompleteTime(byte_position: Int) - UnknownSequence(byte_position: Int, got: String) + UnknownSequence(byte_position: Int, src: String) + UnknownEscapeSequence(byte_position: Int, src: String) // KeyAlreadyInUse(key: List(String)) } @@ -97,8 +101,8 @@ fn new_lexer(src: String) -> Lexer { Splitters( literal_string: splitter.new(["\n", "'"]), multiline_literal_string: splitter.new(["\n", "'''"]), - basic_string: splitter.new(["\n", "\"", "\\\\", "\\"]), - multiline_basic_string: splitter.new(["\n", "\"\"\"", "\\\\", "\\"]), + basic_string: splitter.new(["\n", "\"", "\\"]), + multiline_basic_string: splitter.new(["\n", "\"\"\"", "\\"]), ) Lexer(0, src:, splitters:) } @@ -168,9 +172,12 @@ fn lex(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { "+inf" <> src -> lexed(lexer, src, InfinityToken(Some(Positive))) " " <> _ | "\t" <> _ -> lex_whitespace(lexer) + "#" <> src -> lex_comment(src, lexer) + "'''" <> src -> lex_multiline_literal_string(step(lexer, src)) "'" <> src -> lex_literal_string(step(lexer, src)) - "#" <> src -> lex_comment(src, lexer) + + "\"" <> src -> lex_basic_string(step(lexer, src), "", "") "0" <> src -> lex_number(step(lexer, src), 0, "0") "1" <> src -> lex_number(step(lexer, src), 1, "1") @@ -225,6 +232,7 @@ fn lex_number( "8" <> src -> lex_number(step(lexer, src), int * 10 + 8, text <> "8") "9" <> src -> lex_number(step(lexer, src), int * 10 + 9, text <> "9") + "-" <> src -> lex_date(step(lexer, src), int, text <> "-") ":" <> src if int < 24 -> lex_time_minute(step(lexer, src), int, text <> ":") @@ -268,6 +276,181 @@ fn lex_number( } } +fn lex_date( + lexer: Lexer, + year: Int, + text: String, +) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "01-" <> src -> lex_day(step(lexer, src), year, January, text <> "01-") + "02-" <> src -> lex_day(step(lexer, src), year, February, text <> "02-") + "03-" <> src -> lex_day(step(lexer, src), year, March, text <> "03-") + "04-" <> src -> lex_day(step(lexer, src), year, April, text <> "04-") + "05-" <> src -> lex_day(step(lexer, src), year, May, text <> "05-") + "06-" <> src -> lex_day(step(lexer, src), year, June, text <> "06-") + "07-" <> src -> lex_day(step(lexer, src), year, July, text <> "07-") + "08-" <> src -> lex_day(step(lexer, src), year, August, text <> "08-") + "09-" <> src -> lex_day(step(lexer, src), year, September, text <> "09-") + "10-" <> src -> lex_day(step(lexer, src), year, October, text <> "10-") + "11-" <> src -> lex_day(step(lexer, src), year, November, text <> "11-") + "12-" <> src -> lex_day(step(lexer, src), year, December, text <> "12-") + _ -> Error(IncompleteDate(lexer.position)) + } +} + +fn lex_day( + lexer: Lexer, + year: Int, + month: Month, + text: String, +) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "01" <> src -> lex_date_end(step(lexer, src), year, month, 1, text <> "01") + "02" <> src -> lex_date_end(step(lexer, src), year, month, 2, text <> "02") + "03" <> src -> lex_date_end(step(lexer, src), year, month, 3, text <> "03") + "04" <> src -> lex_date_end(step(lexer, src), year, month, 4, text <> "04") + "05" <> src -> lex_date_end(step(lexer, src), year, month, 5, text <> "05") + "06" <> src -> lex_date_end(step(lexer, src), year, month, 6, text <> "06") + "07" <> src -> lex_date_end(step(lexer, src), year, month, 7, text <> "07") + "08" <> src -> lex_date_end(step(lexer, src), year, month, 8, text <> "08") + "09" <> src -> lex_date_end(step(lexer, src), year, month, 9, text <> "09") + "10" <> src -> lex_date_end(step(lexer, src), year, month, 10, text <> "10") + "11" <> src -> lex_date_end(step(lexer, src), year, month, 11, text <> "11") + "12" <> src -> lex_date_end(step(lexer, src), year, month, 12, text <> "12") + "13" <> src -> lex_date_end(step(lexer, src), year, month, 13, text <> "13") + "14" <> src -> lex_date_end(step(lexer, src), year, month, 14, text <> "14") + "15" <> src -> lex_date_end(step(lexer, src), year, month, 15, text <> "15") + "16" <> src -> lex_date_end(step(lexer, src), year, month, 16, text <> "16") + "17" <> src -> lex_date_end(step(lexer, src), year, month, 17, text <> "17") + "18" <> src -> lex_date_end(step(lexer, src), year, month, 18, text <> "18") + "19" <> src -> lex_date_end(step(lexer, src), year, month, 19, text <> "19") + "20" <> src -> lex_date_end(step(lexer, src), year, month, 20, text <> "20") + "21" <> src -> lex_date_end(step(lexer, src), year, month, 21, text <> "21") + "22" <> src -> lex_date_end(step(lexer, src), year, month, 22, text <> "22") + "23" <> src -> lex_date_end(step(lexer, src), year, month, 23, text <> "23") + "24" <> src -> lex_date_end(step(lexer, src), year, month, 24, text <> "24") + "25" <> src -> lex_date_end(step(lexer, src), year, month, 25, text <> "25") + "26" <> src -> lex_date_end(step(lexer, src), year, month, 26, text <> "26") + "27" <> src -> lex_date_end(step(lexer, src), year, month, 27, text <> "27") + "28" <> src -> lex_date_end(step(lexer, src), year, month, 28, text <> "28") + "29" <> src -> lex_date_end(step(lexer, src), year, month, 29, text <> "29") + "30" <> src -> lex_date_end(step(lexer, src), year, month, 30, text <> "30") + "31" <> src -> lex_date_end(step(lexer, src), year, month, 31, text <> "31") + _ -> Error(IncompleteDate(lexer.position)) + } +} + +fn lex_date_end( + lexer: Lexer, + year: Int, + month: Month, + day: Int, + text: String, +) -> Result(#(Lexer, Token), TomlError) { + let date = calendar.Date(year, month, day) + case lexer.src { + " " as delimeter <> src | "T" as delimeter <> src -> { + let lexer = step(lexer, src) + let text = text <> delimeter + use #(lexer, text, time) <- result.try(lex_time_value(lexer, text)) + lex_datetime_offset(lexer, date, time, text) + } + + _ -> Ok(#(lexer, LocalDateToken(text, date))) + } +} + +fn lex_datetime_offset( + lexer: Lexer, + date: calendar.Date, + time: calendar.TimeOfDay, + text: String, +) -> Result(#(Lexer, Token), TomlError) { + case lexer.src { + "Z" <> src -> { + let lexer = step(lexer, src) + let text = text <> "Z" + let offset = calendar.utc_offset + let token = OffsetDateTimeToken(src: text, date:, time:, offset:) + Ok(#(lexer, token)) + } + "+" <> src -> { + let lexer = step(lexer, src) + let text = text <> "+" + use #(lexer, text, offset) <- result.try(lex_offset(lexer, text, Positive)) + let token = OffsetDateTimeToken(src: text, date:, time:, offset:) + Ok(#(lexer, token)) + } + "-" <> src -> { + let lexer = step(lexer, src) + let text = text <> "-" + use #(lexer, text, offset) <- result.try(lex_offset(lexer, text, Negative)) + let token = OffsetDateTimeToken(src: text, date:, time:, offset:) + Ok(#(lexer, token)) + } + + _ -> Ok(#(lexer, LocalDateTimeToken(text, date:, time:))) + } +} + +fn lex_offset( + lexer: Lexer, + text: String, + sign: Sign, +) -> Result(#(Lexer, String, Duration), TomlError) { + use #(lexer, text, hours, minutes) <- result.try(lex_hour_minute(lexer, text)) + let duration = case sign { + Positive -> duration.add(duration.hours(hours), duration.minutes(minutes)) + Negative -> duration.add(duration.hours(-hours), duration.minutes(-minutes)) + } + Ok(#(lexer, text, duration)) +} + +fn lex_time_value( + lexer: Lexer, + text: String, +) -> Result(#(Lexer, String, calendar.TimeOfDay), TomlError) { + use #(lexer, text, hours, minutes) <- result.try(lex_hour_minute(lexer, text)) + use #(lexer, text, seconds, ns) <- result.try(lex_seconds(lexer, text)) + let time = calendar.TimeOfDay(hours, minutes, seconds, ns) + Ok(#(lexer, text, time)) +} + +fn lex_hour_minute( + lexer: Lexer, + text: String, +) -> Result(#(Lexer, String, Int, Int), TomlError) { + use #(lexer, hours, text) <- result.try(case lexer.src { + "00:" as t <> src -> Ok(#(step(lexer, src), 0, text <> t)) + "01:" as t <> src -> Ok(#(step(lexer, src), 1, text <> t)) + "02:" as t <> src -> Ok(#(step(lexer, src), 2, text <> t)) + "03:" as t <> src -> Ok(#(step(lexer, src), 3, text <> t)) + "04:" as t <> src -> Ok(#(step(lexer, src), 4, text <> t)) + "05:" as t <> src -> Ok(#(step(lexer, src), 5, text <> t)) + "06:" as t <> src -> Ok(#(step(lexer, src), 6, text <> t)) + "07:" as t <> src -> Ok(#(step(lexer, src), 7, text <> t)) + "08:" as t <> src -> Ok(#(step(lexer, src), 8, text <> t)) + "09:" as t <> src -> Ok(#(step(lexer, src), 9, text <> t)) + "10:" as t <> src -> Ok(#(step(lexer, src), 10, text <> t)) + "11:" as t <> src -> Ok(#(step(lexer, src), 11, text <> t)) + "12:" as t <> src -> Ok(#(step(lexer, src), 12, text <> t)) + "13:" as t <> src -> Ok(#(step(lexer, src), 13, text <> t)) + "14:" as t <> src -> Ok(#(step(lexer, src), 14, text <> t)) + "15:" as t <> src -> Ok(#(step(lexer, src), 15, text <> t)) + "16:" as t <> src -> Ok(#(step(lexer, src), 16, text <> t)) + "17:" as t <> src -> Ok(#(step(lexer, src), 17, text <> t)) + "18:" as t <> src -> Ok(#(step(lexer, src), 18, text <> t)) + "19:" as t <> src -> Ok(#(step(lexer, src), 19, text <> t)) + "20:" as t <> src -> Ok(#(step(lexer, src), 20, text <> t)) + "21:" as t <> src -> Ok(#(step(lexer, src), 21, text <> t)) + "22:" as t <> src -> Ok(#(step(lexer, src), 22, text <> t)) + "23:" as t <> src -> Ok(#(step(lexer, src), 23, text <> t)) + _ -> Error(IncompleteTime(lexer.position)) + }) + use #(lexer, text, minutes) <- result.try(lex_number_under_60(lexer, text)) + Ok(#(lexer, text, hours, minutes)) +} + fn lex_time_minute( lexer: Lexer, hours: Int, @@ -706,6 +889,54 @@ fn lex_literal_string(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { } } +fn lex_basic_string( + lexer: Lexer, + text: String, + value: String, +) -> Result(#(Lexer, Token), TomlError) { + let start_position = lexer.position - 1 + let #(lexer, before, split) = + run_splitter(lexer, lexer.splitters.basic_string, lexer.src) + let text = text <> before + let value = value <> before + case split { + "\\" -> { + let text = text <> "\\" + case lex_escape(lexer, text, value) { + Ok(#(lexer, text, value)) -> lex_basic_string(lexer, text, value) + Error(e) -> Error(e) + } + } + "\"" -> Ok(#(lexer, BasicStringToken(src: text, value:))) + "\n" -> Error(UnterminatedString(start_position)) + _ -> Error(UnterminatedString(start_position)) + } +} + +fn lex_escape( + lexer: Lexer, + text: String, + value: String, +) -> Result(#(Lexer, String, String), TomlError) { + case lexer.src { + "t" <> src -> Ok(#(step(lexer, src), text <> "t", value <> "\t")) + "e" <> src -> Ok(#(step(lexer, src), text <> "e", value <> "\u{001b}")) + "b" <> src -> Ok(#(step(lexer, src), text <> "b", value <> "\u{0008}")) + "n" <> src -> Ok(#(step(lexer, src), text <> "n", value <> "\n")) + "r" <> src -> Ok(#(step(lexer, src), text <> "r", value <> "\r")) + "f" <> src -> Ok(#(step(lexer, src), text <> "f", value <> "\f")) + "\"" <> src -> Ok(#(step(lexer, src), text <> "\"", value <> "\"")) + "\\" <> src -> Ok(#(step(lexer, src), text <> "\\", value <> "\\")) + _ -> { + let got = case string.pop_grapheme(lexer.src) { + Ok(#(got, _)) -> got + Error(_) -> "" + } + Error(UnknownEscapeSequence(lexer.position, got)) + } + } +} + fn run_splitter( lexer: Lexer, splitter: Splitter, diff --git a/test/tom/tokenise_test.gleam b/test/tom/tokenise_test.gleam index 4257048..548c667 100644 --- a/test/tom/tokenise_test.gleam +++ b/test/tom/tokenise_test.gleam @@ -1,11 +1,12 @@ import gleam/option.{None, Some} import gleam/time/calendar +import gleam/time/duration import tom.{ BareKeyToken, BasicStringToken, BoolToken, CommaToken, CommentToken, DotToken, DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFile, EqualsToken, - FloatToken, IncompleteFloat, IncompleteTime, InfinityToken, IntToken, - LeftBraceToken, LeftBracketToken, LiteralStringToken, LocalDateTimeToken, - LocalDateToken, LocalTimeToken, MultiLineBasicStringToken, + FloatToken, IncompleteDate, IncompleteFloat, IncompleteTime, InfinityToken, + IntToken, LeftBraceToken, LeftBracketToken, LiteralStringToken, + LocalDateTimeToken, LocalDateToken, LocalTimeToken, MultiLineBasicStringToken, MultiLineLiteralStringToken, NanToken, Negative, NewlineToken, OffsetDateTimeToken, Positive, RightBraceToken, RightBracketToken, UnknownSequence, UnterminatedString, WhitespaceToken, @@ -160,6 +161,111 @@ pub fn multiline_literal_string_test() { ]) } +pub fn basic_string_test() { + assert tom.to_tokens("\"Hello\"") + == Ok([ + BasicStringToken(src: "Hello", value: "Hello"), + EndOfFile, + ]) +} + +pub fn basic_string_newline_test() { + assert tom.to_tokens("\"1\n2\"") + == Error(UnterminatedString(byte_position: 0)) +} + +pub fn basic_string_unterminated_test() { + assert tom.to_tokens("\"1") == Error(UnterminatedString(byte_position: 0)) +} + +pub fn basic_string_quote_escape_test() { + assert tom.to_tokens("\"\\\"\"") + == Ok([ + BasicStringToken(src: "\\\"", value: "\""), + EndOfFile, + ]) +} + +pub fn basic_string_backslash_escape_test() { + assert tom.to_tokens("\"\\\\\"") + == Ok([ + BasicStringToken(src: "\\\\", value: "\\"), + EndOfFile, + ]) +} + +pub fn basic_string_backspace_escape_test() { + assert tom.to_tokens("\"\\b\"") + == Ok([ + BasicStringToken(src: "\\b", value: "\u{8}"), + EndOfFile, + ]) +} + +pub fn basic_string_tab_escape_test() { + assert tom.to_tokens("\"\\t\"") + == Ok([ + BasicStringToken(src: "\\t", value: "\t"), + EndOfFile, + ]) +} + +pub fn basic_string_newline_escape_test() { + assert tom.to_tokens("\"\\n\"") + == Ok([ + BasicStringToken(src: "\\n", value: "\n"), + EndOfFile, + ]) +} + +pub fn basic_string_form_feed_escape_test() { + assert tom.to_tokens("\"\\f\"") + == Ok([ + BasicStringToken(src: "\\f", value: "\u{c}"), + EndOfFile, + ]) +} + +pub fn basic_string_carriage_return_escape_test() { + assert tom.to_tokens("\"\\r\"") + == Ok([ + BasicStringToken(src: "\\r", value: "\r"), + EndOfFile, + ]) +} + +pub fn basic_string_escape_escape_test() { + assert tom.to_tokens("\"\\e\"") + == Ok([ + BasicStringToken(src: "\\e", value: "\u{1b}"), + EndOfFile, + ]) +} + +pub fn basic_string_hex_escape_test() { + assert tom.to_tokens("\"\\x7f\"") + == Ok([ + BasicStringToken(src: "\\x7f", value: "\u{7f}"), + EndOfFile, + ]) +} + +pub fn basic_string_unicode_escape_test() { + assert tom.to_tokens("\"\\u03B1\"") + == Ok([ + BasicStringToken(src: "\\u03B1", value: "\u{3b1}"), + EndOfFile, + ]) +} + +pub fn basic_string_unicode_long_escape_test() { + assert tom.to_tokens("\"\\U0001F600\"") + == Ok([ + BasicStringToken(src: "\\U0001F600", value: "\u{1f600}"), + EndOfFile, + ]) +} + pub fn unexpected_test() { assert tom.to_tokens("???") == Error(UnknownSequence(0, "?")) } @@ -287,3 +393,70 @@ pub fn local_time_incomplete_seconds_test() { pub fn local_time_incomplete_fractional_seconds_test() { assert tom.to_tokens("07:32:00.") == Error(IncompleteTime(9)) } + +pub fn local_date_test() { + assert tom.to_tokens("1991-01-05") + == Ok([ + LocalDateToken("1991-01-05", calendar.Date(1991, calendar.January, 5)), + EndOfFile, + ]) +} + +pub fn local_date_incomplete_month_test() { + assert tom.to_tokens("1991-") == Error(IncompleteDate(5)) +} + +pub fn local_date_incomplete_day_test() { + assert tom.to_tokens("1991-01-") == Error(IncompleteDate(8)) +} + +pub fn local_date_time_test() { + assert tom.to_tokens("1991-01-05T07:32:00") + == Ok([ + LocalDateTimeToken( + "1991-01-05T07:32:00", + calendar.Date(1991, calendar.January, 5), + calendar.TimeOfDay(7, 32, 0, 0), + ), + EndOfFile, + ]) +} + +pub fn offset_date_time_test() { + assert tom.to_tokens("1991-01-05T07:32:00Z") + == Ok([ + OffsetDateTimeToken( + "1991-01-05T07:32:00Z", + calendar.Date(1991, calendar.January, 5), + calendar.TimeOfDay(7, 32, 0, 0), + calendar.utc_offset, + ), + EndOfFile, + ]) +} + +pub fn offset_date_time_positive_offset_test() { + assert tom.to_tokens("1991-01-05T07:32:00+07:30") + == Ok([ + OffsetDateTimeToken( + "1991-01-05T07:32:00+07:30", + calendar.Date(1991, calendar.January, 5), + calendar.TimeOfDay(7, 32, 0, 0), + duration.add(duration.hours(7), duration.minutes(30)), + ), + EndOfFile, + ]) +} + +pub fn offset_date_time_negative_offset_test() { + assert tom.to_tokens("1991-01-05T07:32:00-05:15") + == Ok([ + OffsetDateTimeToken( + "1991-01-05T07:32:00-05:15", + calendar.Date(1991, calendar.January, 5), + calendar.TimeOfDay(7, 32, 0, 0), + duration.add(duration.hours(-5), duration.minutes(-15)), + ), + EndOfFile, + ]) +} From 1ead0380cf8f22efbbf03b6590635ec3fd511bab Mon Sep 17 00:00:00 2001 From: Louis Pilfold Date: Mon, 13 Apr 2026 16:46:54 +0100 Subject: [PATCH 6/7] Full lexing! --- README.md | 8 -- src/tom.gleam | 85 ++++++++++++---- test/tom/tokenise_test.gleam | 182 ++++++++++++++++++++++++++++++++++- 3 files changed, 248 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index cca986a..3c5204a 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,3 @@ pub fn main() { ``` Further documentation can be found at . - -## Status - -The following string escape sequences are not supported yet: - -- `\xHH` -- `\uHHHH` -- `\UHHHHHHHH` diff --git a/src/tom.gleam b/src/tom.gleam index ae4f16d..e431c44 100644 --- a/src/tom.gleam +++ b/src/tom.gleam @@ -1,3 +1,4 @@ +import gleam/bool import gleam/float import gleam/int import gleam/list @@ -83,8 +84,9 @@ pub type TomlError { IncompleteFloat(byte_position: Int) IncompleteDate(byte_position: Int) IncompleteTime(byte_position: Int) - UnknownSequence(byte_position: Int, src: String) - UnknownEscapeSequence(byte_position: Int, src: String) + UnknownSequence(byte_position: Int) + InvalidEscapeSequence(byte_position: Int) + UnknownEscapeSequence(byte_position: Int) // KeyAlreadyInUse(key: List(String)) } @@ -100,9 +102,8 @@ fn new_lexer(src: String) -> Lexer { let splitters = Splitters( literal_string: splitter.new(["\n", "'"]), - multiline_literal_string: splitter.new(["\n", "'''"]), basic_string: splitter.new(["\n", "\"", "\\"]), - multiline_basic_string: splitter.new(["\n", "\"\"\"", "\\"]), + multiline_basic_string: splitter.new(["\"\"\"", "\\"]), ) Lexer(0, src:, splitters:) } @@ -120,7 +121,6 @@ type Lexer { type Splitters { Splitters( literal_string: Splitter, - multiline_literal_string: Splitter, basic_string: Splitter, multiline_basic_string: Splitter, ) @@ -177,6 +177,7 @@ fn lex(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { "'''" <> src -> lex_multiline_literal_string(step(lexer, src)) "'" <> src -> lex_literal_string(step(lexer, src)) + "\"\"\"" <> src -> lex_multiline_basic_string(step(lexer, src), "", "") "\"" <> src -> lex_basic_string(step(lexer, src), "", "") "0" <> src -> lex_number(step(lexer, src), 0, "0") @@ -833,13 +834,7 @@ fn lex_bare_key( | "_" as c <> src -> lex_bare_key(step(lexer, src), content <> c) src if content != "" -> lexed(lexer, src, BareKeyToken(content)) - src -> { - let got = case string.pop_grapheme(src) { - Ok(#(got, _)) -> got - Error(_) -> "" - } - Error(UnknownSequence(lexer.position, got)) - } + _ -> Error(UnknownSequence(lexer.position)) } } @@ -913,12 +908,42 @@ fn lex_basic_string( } } +fn lex_multiline_basic_string( + lexer: Lexer, + text: String, + value: String, +) -> Result(#(Lexer, Token), TomlError) { + let start_position = lexer.position - 3 + let #(lexer, before, split) = + run_splitter(lexer, lexer.splitters.multiline_basic_string, lexer.src) + let text = text <> before + let value = value <> before + case split { + "\\" -> { + let text = text <> "\\" + case lex_escape(lexer, text, value) { + Ok(#(lexer, text, value)) -> + lex_multiline_basic_string(lexer, text, value) + Error(e) -> Error(e) + } + } + "\"\"\"" -> { + let value = drop_leading_newline(value) + Ok(#(lexer, MultiLineBasicStringToken(src: text, value:))) + } + _ -> Error(UnterminatedString(start_position)) + } +} + fn lex_escape( lexer: Lexer, text: String, value: String, ) -> Result(#(Lexer, String, String), TomlError) { case lexer.src { + "x" <> src -> lex_unicode_escape(step(lexer, src), text <> "x", value, 2) + "u" <> src -> lex_unicode_escape(step(lexer, src), text <> "u", value, 4) + "U" <> src -> lex_unicode_escape(step(lexer, src), text <> "U", value, 8) "t" <> src -> Ok(#(step(lexer, src), text <> "t", value <> "\t")) "e" <> src -> Ok(#(step(lexer, src), text <> "e", value <> "\u{001b}")) "b" <> src -> Ok(#(step(lexer, src), text <> "b", value <> "\u{0008}")) @@ -927,16 +952,40 @@ fn lex_escape( "f" <> src -> Ok(#(step(lexer, src), text <> "f", value <> "\f")) "\"" <> src -> Ok(#(step(lexer, src), text <> "\"", value <> "\"")) "\\" <> src -> Ok(#(step(lexer, src), text <> "\\", value <> "\\")) - _ -> { - let got = case string.pop_grapheme(lexer.src) { - Ok(#(got, _)) -> got - Error(_) -> "" - } - Error(UnknownEscapeSequence(lexer.position, got)) + "\n" <> src -> { + let #(whitespace, src) = take_whitespace("\n", src) + let lexer = step(lexer, src) + Ok(#(lexer, text <> whitespace, value)) } + _ -> Error(UnknownEscapeSequence(lexer.position)) } } +fn lex_unicode_escape( + lexer: Lexer, + text: String, + value: String, + digits: Int, +) -> Result(#(Lexer, String, String), TomlError) { + let hex = string.slice(lexer.src, 0, digits) + use <- bool.guard( + when: string.byte_size(hex) != digits, + return: Error(InvalidEscapeSequence(lexer.position)), + ) + let src = string.slice(lexer.src, digits, string.byte_size(lexer.src)) + + use codepoint <- result.try( + int.base_parse(hex, 16) + |> result.try(string.utf_codepoint) + |> result.replace_error(InvalidEscapeSequence(lexer.position)), + ) + + let lexer = step(lexer, src) + let text = text <> hex + let value = value <> string.from_utf_codepoints([codepoint]) + Ok(#(lexer, text, value)) +} + fn run_splitter( lexer: Lexer, splitter: Splitter, diff --git a/test/tom/tokenise_test.gleam b/test/tom/tokenise_test.gleam index 548c667..2ac2ba2 100644 --- a/test/tom/tokenise_test.gleam +++ b/test/tom/tokenise_test.gleam @@ -250,6 +250,14 @@ pub fn basic_string_hex_escape_test() { ]) } +pub fn basic_string_hex_escape_upper_range_test() { + assert tom.to_tokens("\"\\xff\"") + == Ok([ + BasicStringToken(src: "\\xff", value: "\u{ff}"), + EndOfFile, + ]) +} + pub fn basic_string_unicode_escape_test() { assert tom.to_tokens("\"\\u03B1\"") == Ok([ @@ -266,8 +274,180 @@ pub fn basic_string_unicode_long_escape_test() { ]) } +pub fn multiline_basic_string_test() { + assert tom.to_tokens("\"\"\"\nHello\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\nHello", value: "Hello"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_no_newline_test() { + assert tom.to_tokens("\"\"\"Hello\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "Hello", value: "Hello"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_multiline_test() { + assert tom.to_tokens("\"\"\"\n1\n2\n3\n\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n1\n2\n3\n", value: "1\n2\n3\n"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_newline_test() { + assert tom.to_tokens("\"\"\"\n1\n2\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n1\n2", value: "1\n2"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_unterminated_test() { + assert tom.to_tokens("\"\"\"\n1") + == Error(UnterminatedString(byte_position: 0)) +} + +pub fn multiline_basic_string_quote_escape_test() { + assert tom.to_tokens("\"\"\"\n\\\"\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\\"", value: "\""), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_backslash_escape_test() { + assert tom.to_tokens("\"\"\"\n\\\\\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\\\", value: "\\"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_backspace_escape_test() { + assert tom.to_tokens("\"\"\"\n\\b\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\b", value: "\u{8}"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_tab_escape_test() { + assert tom.to_tokens("\"\"\"\n\\t\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\t", value: "\t"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_newline_escape_test() { + assert tom.to_tokens("\"\"\"\n\\n\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\n", value: "\n"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_form_feed_escape_test() { + assert tom.to_tokens("\"\"\"\n\\f\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\f", value: "\u{c}"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_carriage_return_escape_test() { + assert tom.to_tokens("\"\"\"\n\\r\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\r", value: "\r"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_escape_escape_test() { + assert tom.to_tokens("\"\"\"\n\\e\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\e", value: "\u{1b}"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_hex_escape_test() { + assert tom.to_tokens("\"\"\"\n\\x7f\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\x7f", value: "\u{7f}"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_hex_escape_upper_range_test() { + assert tom.to_tokens("\"\"\"\n\\xff\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\xff", value: "\u{ff}"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_unicode_escape_test() { + assert tom.to_tokens("\"\"\"\n\\u03B1\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\u03B1", value: "\u{3b1}"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_unicode_long_escape_test() { + assert tom.to_tokens("\"\"\"\n\\U0001F600\"\"\"") + == Ok([ + MultiLineBasicStringToken(src: "\n\\U0001F600", value: "\u{1f600}"), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_line_ending_backslash_test() { + assert tom.to_tokens( + "\"\"\"\nThe quick brown \\\nfox jumps over \\\nthe lazy dog.\"\"\"", + ) + == Ok([ + MultiLineBasicStringToken( + src: "\nThe quick brown \\\nfox jumps over \\\nthe lazy dog.", + value: "The quick brown fox jumps over the lazy dog.", + ), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_line_ending_backslash_spaces_test() { + assert tom.to_tokens( + "\"\"\"\nThe quick brown \\\n fox jumps over \\\n the lazy dog.\"\"\"", + ) + == Ok([ + MultiLineBasicStringToken( + src: "\nThe quick brown \\\n fox jumps over \\\n the lazy dog.", + value: "The quick brown fox jumps over the lazy dog.", + ), + EndOfFile, + ]) +} + +pub fn multiline_basic_string_line_ending_backslash_tabs_test() { + assert tom.to_tokens( + "\"\"\"\nThe quick brown \\\n\tfox jumps over \\\n\tthe lazy dog.\"\"\"", + ) + == Ok([ + MultiLineBasicStringToken( + src: "\nThe quick brown \\\n\tfox jumps over \\\n\tthe lazy dog.", + value: "The quick brown fox jumps over the lazy dog.", + ), + EndOfFile, + ]) +} + pub fn unexpected_test() { - assert tom.to_tokens("???") == Error(UnknownSequence(0, "?")) + assert tom.to_tokens("???") == Error(UnknownSequence(0)) } pub fn key_test() { From f3d81e49916d26ef786fe6ca263607f1835799e0 Mon Sep 17 00:00:00 2001 From: Louis Pilfold Date: Thu, 30 Apr 2026 11:27:00 +0100 Subject: [PATCH 7/7] Symbols --- src/tom.gleam | 702 +++++++++++------- test/tom/to_symbols_test.gleam | 16 + ...kenise_test.gleam => to_tokens_test.gleam} | 167 +++-- 3 files changed, 555 insertions(+), 330 deletions(-) create mode 100644 test/tom/to_symbols_test.gleam rename test/tom/{tokenise_test.gleam => to_tokens_test.gleam} (82%) diff --git a/src/tom.gleam b/src/tom.gleam index e431c44..04b970d 100644 --- a/src/tom.gleam +++ b/src/tom.gleam @@ -70,7 +70,69 @@ pub type Token { /// A time. LocalTimeToken(src: String, time: calendar.TimeOfDay) /// The end! - EndOfFile + EndOfFileToken +} + +pub type Symbol { + WhitespaceSymbol(postition: Int, src: String) + NewlineSymbol(position: Int) + /// A comment starting with `#` and ending with `\n`. + CommentSymbol(position: Int, src: String) + /// `=` + EqualsSymbol(position: Int) + /// `,` + CommaSymbol(position: Int) + /// `{` + InlineTableStartSymbol(position: Int) + /// `}` + InlineTableEndSymbol(position: Int) + /// `[` + ArrayStartSymbol(position: Int) + /// `]` + ArrayEndSymbol(position: Int) + /// `[wibble]` + TableHeader(position: Int) + /// `[[wibble]]` + ArrayTableHeader(position: Int) + /// An key + KeySymbol(position: Int, value: String) + /// A double-quote single-line string. + BasicStringSymbol(position: Int, src: String, value: String) + /// A double-quote multi-line string. + MultiLineBasicStringSymbol(position: Int, src: String, value: String) + /// A single-quote single-line string. + LiteralStringSymbol(position: Int, src: String) + /// A single-quote multi-line string. + MultiLineLiteralStringSymbol(position: Int, src: String, value: String) + /// An int e.g `123` + IntSymbol(position: Int, src: String, value: Int) + /// A float literal e.g. `123.456` + FloatSymbol(position: Int, src: String, value: Float) + /// inf, -inf, +inf + InfinitySymbol(position: Int, sign: Option(Sign)) + /// nan, -nan, +nan + NanSymbol(position: Int, sign: Option(Sign)) + /// `true` or `false`. + BoolSymbol(position: Int, value: Bool) + /// A date-time with an offset. + OffsetDateTimeSymbol( + position: Int, + src: String, + date: calendar.Date, + time: calendar.TimeOfDay, + offset: Duration, + ) + /// A date-time with no offset. + LocalDateTimeSymbol( + position: Int, + src: String, + date: calendar.Date, + time: calendar.TimeOfDay, + ) + /// A date. + LocalDateSymbol(position: Int, src: String, date: calendar.Date) + /// A time. + LocalTimeSymbol(position: Int, src: String, time: calendar.TimeOfDay) } pub type Sign { @@ -97,6 +159,13 @@ pub fn to_tokens(src: String) -> Result(List(Token), TomlError) { |> result.map(list.reverse) } +// TODO: document +pub fn to_symbols(src: String) -> Result(List(Symbol), TomlError) { + new_lexer(src) + |> fold_symbols([], fn(symbols, symbol) { Ok([symbol, ..symbols]) }) + |> result.map(list.reverse) +} + fn new_lexer(src: String) -> Lexer { let src = string.replace(src, "\r\n", "\n") let splitters = @@ -108,7 +177,7 @@ fn new_lexer(src: String) -> Lexer { Lexer(0, src:, splitters:) } -fn step(lexer: Lexer, src: String) -> Lexer { +fn lex_step(lexer: Lexer, src: String) -> Lexer { let position = lexer.position + string.byte_size(lexer.src) - string.byte_size(src) Lexer(..lexer, position:, src:) @@ -118,6 +187,85 @@ type Lexer { Lexer(position: Int, src: String, splitters: Splitters) } +fn symbolise(lexer: Lexer) -> Result(#(Lexer, List(Symbol)), TomlError) { + case lex(lexer) { + Ok(#(lexer, token)) -> + case token { + NewlineToken -> Ok(#(lexer, [NewlineSymbol(lexer.position)])) + WhitespaceToken(src) -> + Ok(#(lexer, [WhitespaceSymbol(lexer.position, src)])) + + BareKeyToken(value:) -> symbolise_key(lexer, [value], value) + + CommentToken(_) -> todo + EqualsToken -> todo + DotToken -> todo + CommaToken -> todo + LeftBraceToken -> todo + RightBraceToken -> todo + LeftBracketToken -> todo + RightBracketToken -> todo + DoubleLeftBracketToken -> todo + DoubleRightBracketToken -> todo + BasicStringToken(src:, value:) -> todo + MultiLineBasicStringToken(src:, value:) -> todo + LiteralStringToken(src:) -> todo + MultiLineLiteralStringToken(src:, value:) -> todo + IntToken(src:, value:) -> todo + FloatToken(src:, value:) -> todo + InfinityToken(sign:) -> todo + NanToken(sign:) -> todo + BoolToken(value:) -> todo + OffsetDateTimeToken(src:, date:, time:, offset:) -> todo + LocalDateTimeToken(src:, date:, time:) -> todo + LocalDateToken(src:, date:) -> todo + LocalTimeToken(src:, time:) -> todo + EndOfFileToken -> todo + } + Error(e) -> Error(e) + } +} + +fn symbolise_key( + lexer: Lexer, + segments: List(String), + src: String, +) -> Result(#(Lexer, List(Symbol)), TomlError) { + case lex(lexer) { + Error(e) -> Error(e) + Ok(#(lexer, BareKeyToken(value:))) -> + symbolise_key(lexer, [value, ..segments], src <> value) + + Ok(#(lexer, token)) -> todo as string.inspect(token) + } + // WhitespaceToken(_) -> todo + // NewlineToken -> todo + // CommentToken(_) -> todo + // EqualsToken -> todo + // DotToken -> todo + // CommaToken -> todo + // LeftBraceToken -> todo + // RightBraceToken -> todo + // LeftBracketToken -> todo + // RightBracketToken -> todo + // DoubleLeftBracketToken -> todo + // DoubleRightBracketToken -> todo + // BasicStringToken(src:, value:) -> todo + // MultiLineBasicStringToken(src:, value:) -> todo + // LiteralStringToken(src:) -> todo + // MultiLineLiteralStringToken(src:, value:) -> todo + // IntToken(src:, value:) -> todo + // FloatToken(src:, value:) -> todo + // InfinityToken(sign:) -> todo + // NanToken(sign:) -> todo + // BoolToken(value:) -> todo + // OffsetDateTimeToken(src:, date:, time:, offset:) -> todo + // LocalDateTimeToken(src:, date:, time:) -> todo + // LocalDateToken(src:, date:) -> todo + // LocalTimeToken(src:, time:) -> todo + // EndOfFileToken -> todo +} + type Splitters { Splitters( literal_string: Splitter, @@ -126,13 +274,30 @@ type Splitters { ) } +fn fold_symbols( + lexer: Lexer, + output: output, + reduce: fn(output, Symbol) -> Result(output, TomlError), +) -> Result(output, TomlError) { + case symbolise(lexer) { + Ok(#(_lexer, [])) -> Ok(output) + Ok(#(lexer, symbols)) -> { + case list.try_fold(symbols, output, reduce) { + Ok(output) -> fold_symbols(lexer, output, reduce) + Error(error) -> Error(error) + } + } + Error(error) -> Error(error) + } +} + fn fold_tokens( lexer: Lexer, output: output, reduce: fn(output, Int, Token) -> Result(output, TomlError), ) -> Result(output, TomlError) { case lex(lexer) { - Ok(#(lexer, EndOfFile as token)) -> { + Ok(#(lexer, EndOfFileToken as token)) -> { reduce(output, lexer.position, token) } Ok(#(lexer, token)) -> { @@ -147,7 +312,7 @@ fn fold_tokens( fn lex(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { case lexer.src { - "" -> Ok(#(lexer, EndOfFile)) + "" -> Ok(#(lexer, EndOfFileToken)) "[[" <> src -> lexed(lexer, src, DoubleLeftBracketToken) "]]" <> src -> lexed(lexer, src, DoubleRightBracketToken) @@ -174,42 +339,42 @@ fn lex(lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { " " <> _ | "\t" <> _ -> lex_whitespace(lexer) "#" <> src -> lex_comment(src, lexer) - "'''" <> src -> lex_multiline_literal_string(step(lexer, src)) - "'" <> src -> lex_literal_string(step(lexer, src)) - - "\"\"\"" <> src -> lex_multiline_basic_string(step(lexer, src), "", "") - "\"" <> src -> lex_basic_string(step(lexer, src), "", "") - - "0" <> src -> lex_number(step(lexer, src), 0, "0") - "1" <> src -> lex_number(step(lexer, src), 1, "1") - "2" <> src -> lex_number(step(lexer, src), 2, "2") - "3" <> src -> lex_number(step(lexer, src), 3, "3") - "4" <> src -> lex_number(step(lexer, src), 4, "4") - "5" <> src -> lex_number(step(lexer, src), 5, "5") - "6" <> src -> lex_number(step(lexer, src), 6, "6") - "7" <> src -> lex_number(step(lexer, src), 7, "7") - "8" <> src -> lex_number(step(lexer, src), 8, "8") - "9" <> src -> lex_number(step(lexer, src), 9, "9") - "+0" <> src -> lex_number(step(lexer, src), 0, "+0") - "+1" <> src -> lex_number(step(lexer, src), 1, "+1") - "+2" <> src -> lex_number(step(lexer, src), 2, "+2") - "+3" <> src -> lex_number(step(lexer, src), 3, "+3") - "+4" <> src -> lex_number(step(lexer, src), 4, "+4") - "+5" <> src -> lex_number(step(lexer, src), 5, "+5") - "+6" <> src -> lex_number(step(lexer, src), 6, "+6") - "+7" <> src -> lex_number(step(lexer, src), 7, "+7") - "+8" <> src -> lex_number(step(lexer, src), 8, "+8") - "+9" <> src -> lex_number(step(lexer, src), 9, "+9") - "-0" <> src -> lex_number(step(lexer, src), 0, "-0") - "-1" <> src -> lex_number(step(lexer, src), 1, "-1") - "-2" <> src -> lex_number(step(lexer, src), 2, "-2") - "-3" <> src -> lex_number(step(lexer, src), 3, "-3") - "-4" <> src -> lex_number(step(lexer, src), 4, "-4") - "-5" <> src -> lex_number(step(lexer, src), 5, "-5") - "-6" <> src -> lex_number(step(lexer, src), 6, "-6") - "-7" <> src -> lex_number(step(lexer, src), 7, "-7") - "-8" <> src -> lex_number(step(lexer, src), 8, "-8") - "-9" <> src -> lex_number(step(lexer, src), 9, "-9") + "'''" <> src -> lex_multiline_literal_string(lex_step(lexer, src)) + "'" <> src -> lex_literal_string(lex_step(lexer, src)) + + "\"\"\"" <> src -> lex_multiline_basic_string(lex_step(lexer, src), "", "") + "\"" <> src -> lex_basic_string(lex_step(lexer, src), "", "") + + "0" <> src -> lex_number(lex_step(lexer, src), 0, "0") + "1" <> src -> lex_number(lex_step(lexer, src), 1, "1") + "2" <> src -> lex_number(lex_step(lexer, src), 2, "2") + "3" <> src -> lex_number(lex_step(lexer, src), 3, "3") + "4" <> src -> lex_number(lex_step(lexer, src), 4, "4") + "5" <> src -> lex_number(lex_step(lexer, src), 5, "5") + "6" <> src -> lex_number(lex_step(lexer, src), 6, "6") + "7" <> src -> lex_number(lex_step(lexer, src), 7, "7") + "8" <> src -> lex_number(lex_step(lexer, src), 8, "8") + "9" <> src -> lex_number(lex_step(lexer, src), 9, "9") + "+0" <> src -> lex_number(lex_step(lexer, src), 0, "+0") + "+1" <> src -> lex_number(lex_step(lexer, src), 1, "+1") + "+2" <> src -> lex_number(lex_step(lexer, src), 2, "+2") + "+3" <> src -> lex_number(lex_step(lexer, src), 3, "+3") + "+4" <> src -> lex_number(lex_step(lexer, src), 4, "+4") + "+5" <> src -> lex_number(lex_step(lexer, src), 5, "+5") + "+6" <> src -> lex_number(lex_step(lexer, src), 6, "+6") + "+7" <> src -> lex_number(lex_step(lexer, src), 7, "+7") + "+8" <> src -> lex_number(lex_step(lexer, src), 8, "+8") + "+9" <> src -> lex_number(lex_step(lexer, src), 9, "+9") + "-0" <> src -> lex_number(lex_step(lexer, src), 0, "-0") + "-1" <> src -> lex_number(lex_step(lexer, src), 1, "-1") + "-2" <> src -> lex_number(lex_step(lexer, src), 2, "-2") + "-3" <> src -> lex_number(lex_step(lexer, src), 3, "-3") + "-4" <> src -> lex_number(lex_step(lexer, src), 4, "-4") + "-5" <> src -> lex_number(lex_step(lexer, src), 5, "-5") + "-6" <> src -> lex_number(lex_step(lexer, src), 6, "-6") + "-7" <> src -> lex_number(lex_step(lexer, src), 7, "-7") + "-8" <> src -> lex_number(lex_step(lexer, src), 8, "-8") + "-9" <> src -> lex_number(lex_step(lexer, src), 9, "-9") _ -> lex_bare_key(lexer, "") } @@ -221,50 +386,50 @@ fn lex_number( text: String, ) -> Result(#(Lexer, Token), TomlError) { case lexer.src { - "_" <> src -> lex_number(step(lexer, src), int, text <> "_") - "0" <> src -> lex_number(step(lexer, src), int * 10 + 0, text <> "0") - "1" <> src -> lex_number(step(lexer, src), int * 10 + 1, text <> "1") - "2" <> src -> lex_number(step(lexer, src), int * 10 + 2, text <> "2") - "3" <> src -> lex_number(step(lexer, src), int * 10 + 3, text <> "3") - "4" <> src -> lex_number(step(lexer, src), int * 10 + 4, text <> "4") - "5" <> src -> lex_number(step(lexer, src), int * 10 + 5, text <> "5") - "6" <> src -> lex_number(step(lexer, src), int * 10 + 6, text <> "6") - "7" <> src -> lex_number(step(lexer, src), int * 10 + 7, text <> "7") - "8" <> src -> lex_number(step(lexer, src), int * 10 + 8, text <> "8") - "9" <> src -> lex_number(step(lexer, src), int * 10 + 9, text <> "9") - - "-" <> src -> lex_date(step(lexer, src), int, text <> "-") + "_" <> src -> lex_number(lex_step(lexer, src), int, text <> "_") + "0" <> src -> lex_number(lex_step(lexer, src), int * 10 + 0, text <> "0") + "1" <> src -> lex_number(lex_step(lexer, src), int * 10 + 1, text <> "1") + "2" <> src -> lex_number(lex_step(lexer, src), int * 10 + 2, text <> "2") + "3" <> src -> lex_number(lex_step(lexer, src), int * 10 + 3, text <> "3") + "4" <> src -> lex_number(lex_step(lexer, src), int * 10 + 4, text <> "4") + "5" <> src -> lex_number(lex_step(lexer, src), int * 10 + 5, text <> "5") + "6" <> src -> lex_number(lex_step(lexer, src), int * 10 + 6, text <> "6") + "7" <> src -> lex_number(lex_step(lexer, src), int * 10 + 7, text <> "7") + "8" <> src -> lex_number(lex_step(lexer, src), int * 10 + 8, text <> "8") + "9" <> src -> lex_number(lex_step(lexer, src), int * 10 + 9, text <> "9") + + "-" <> src -> lex_date(lex_step(lexer, src), int, text <> "-") ":" <> src if int < 24 -> - lex_time_minute(step(lexer, src), int, text <> ":") + lex_time_minute(lex_step(lexer, src), int, text <> ":") "." <> src -> { let float = int.to_float(int) - lex_float(step(lexer, src), float, 0.1, text <> ".") + lex_float(lex_step(lexer, src), float, 0.1, text <> ".") } "e+" <> src -> { let float = int.to_float(int) - lex_exponent(step(lexer, src), float, text <> "e+", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "e+", 0, Positive) } "e-" <> src -> { let float = int.to_float(int) - lex_exponent(step(lexer, src), float, text <> "e-", 0, Negative) + lex_exponent(lex_step(lexer, src), float, text <> "e-", 0, Negative) } "e" <> src -> { let float = int.to_float(int) - lex_exponent(step(lexer, src), float, text <> "e", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "e", 0, Positive) } "E+" <> src -> { let float = int.to_float(int) - lex_exponent(step(lexer, src), float, text <> "E+", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "E+", 0, Positive) } "E-" <> src -> { let float = int.to_float(int) - lex_exponent(step(lexer, src), float, text <> "E-", 0, Negative) + lex_exponent(lex_step(lexer, src), float, text <> "E-", 0, Negative) } "E" <> src -> { let float = int.to_float(int) - lex_exponent(step(lexer, src), float, text <> "E", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "E", 0, Positive) } src -> { @@ -283,18 +448,19 @@ fn lex_date( text: String, ) -> Result(#(Lexer, Token), TomlError) { case lexer.src { - "01-" <> src -> lex_day(step(lexer, src), year, January, text <> "01-") - "02-" <> src -> lex_day(step(lexer, src), year, February, text <> "02-") - "03-" <> src -> lex_day(step(lexer, src), year, March, text <> "03-") - "04-" <> src -> lex_day(step(lexer, src), year, April, text <> "04-") - "05-" <> src -> lex_day(step(lexer, src), year, May, text <> "05-") - "06-" <> src -> lex_day(step(lexer, src), year, June, text <> "06-") - "07-" <> src -> lex_day(step(lexer, src), year, July, text <> "07-") - "08-" <> src -> lex_day(step(lexer, src), year, August, text <> "08-") - "09-" <> src -> lex_day(step(lexer, src), year, September, text <> "09-") - "10-" <> src -> lex_day(step(lexer, src), year, October, text <> "10-") - "11-" <> src -> lex_day(step(lexer, src), year, November, text <> "11-") - "12-" <> src -> lex_day(step(lexer, src), year, December, text <> "12-") + "01-" <> src -> lex_day(lex_step(lexer, src), year, January, text <> "01-") + "02-" <> src -> lex_day(lex_step(lexer, src), year, February, text <> "02-") + "03-" <> src -> lex_day(lex_step(lexer, src), year, March, text <> "03-") + "04-" <> src -> lex_day(lex_step(lexer, src), year, April, text <> "04-") + "05-" <> src -> lex_day(lex_step(lexer, src), year, May, text <> "05-") + "06-" <> src -> lex_day(lex_step(lexer, src), year, June, text <> "06-") + "07-" <> src -> lex_day(lex_step(lexer, src), year, July, text <> "07-") + "08-" <> src -> lex_day(lex_step(lexer, src), year, August, text <> "08-") + "09-" <> src -> + lex_day(lex_step(lexer, src), year, September, text <> "09-") + "10-" <> src -> lex_day(lex_step(lexer, src), year, October, text <> "10-") + "11-" <> src -> lex_day(lex_step(lexer, src), year, November, text <> "11-") + "12-" <> src -> lex_day(lex_step(lexer, src), year, December, text <> "12-") _ -> Error(IncompleteDate(lexer.position)) } } @@ -306,37 +472,68 @@ fn lex_day( text: String, ) -> Result(#(Lexer, Token), TomlError) { case lexer.src { - "01" <> src -> lex_date_end(step(lexer, src), year, month, 1, text <> "01") - "02" <> src -> lex_date_end(step(lexer, src), year, month, 2, text <> "02") - "03" <> src -> lex_date_end(step(lexer, src), year, month, 3, text <> "03") - "04" <> src -> lex_date_end(step(lexer, src), year, month, 4, text <> "04") - "05" <> src -> lex_date_end(step(lexer, src), year, month, 5, text <> "05") - "06" <> src -> lex_date_end(step(lexer, src), year, month, 6, text <> "06") - "07" <> src -> lex_date_end(step(lexer, src), year, month, 7, text <> "07") - "08" <> src -> lex_date_end(step(lexer, src), year, month, 8, text <> "08") - "09" <> src -> lex_date_end(step(lexer, src), year, month, 9, text <> "09") - "10" <> src -> lex_date_end(step(lexer, src), year, month, 10, text <> "10") - "11" <> src -> lex_date_end(step(lexer, src), year, month, 11, text <> "11") - "12" <> src -> lex_date_end(step(lexer, src), year, month, 12, text <> "12") - "13" <> src -> lex_date_end(step(lexer, src), year, month, 13, text <> "13") - "14" <> src -> lex_date_end(step(lexer, src), year, month, 14, text <> "14") - "15" <> src -> lex_date_end(step(lexer, src), year, month, 15, text <> "15") - "16" <> src -> lex_date_end(step(lexer, src), year, month, 16, text <> "16") - "17" <> src -> lex_date_end(step(lexer, src), year, month, 17, text <> "17") - "18" <> src -> lex_date_end(step(lexer, src), year, month, 18, text <> "18") - "19" <> src -> lex_date_end(step(lexer, src), year, month, 19, text <> "19") - "20" <> src -> lex_date_end(step(lexer, src), year, month, 20, text <> "20") - "21" <> src -> lex_date_end(step(lexer, src), year, month, 21, text <> "21") - "22" <> src -> lex_date_end(step(lexer, src), year, month, 22, text <> "22") - "23" <> src -> lex_date_end(step(lexer, src), year, month, 23, text <> "23") - "24" <> src -> lex_date_end(step(lexer, src), year, month, 24, text <> "24") - "25" <> src -> lex_date_end(step(lexer, src), year, month, 25, text <> "25") - "26" <> src -> lex_date_end(step(lexer, src), year, month, 26, text <> "26") - "27" <> src -> lex_date_end(step(lexer, src), year, month, 27, text <> "27") - "28" <> src -> lex_date_end(step(lexer, src), year, month, 28, text <> "28") - "29" <> src -> lex_date_end(step(lexer, src), year, month, 29, text <> "29") - "30" <> src -> lex_date_end(step(lexer, src), year, month, 30, text <> "30") - "31" <> src -> lex_date_end(step(lexer, src), year, month, 31, text <> "31") + "01" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 1, text <> "01") + "02" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 2, text <> "02") + "03" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 3, text <> "03") + "04" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 4, text <> "04") + "05" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 5, text <> "05") + "06" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 6, text <> "06") + "07" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 7, text <> "07") + "08" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 8, text <> "08") + "09" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 9, text <> "09") + "10" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 10, text <> "10") + "11" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 11, text <> "11") + "12" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 12, text <> "12") + "13" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 13, text <> "13") + "14" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 14, text <> "14") + "15" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 15, text <> "15") + "16" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 16, text <> "16") + "17" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 17, text <> "17") + "18" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 18, text <> "18") + "19" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 19, text <> "19") + "20" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 20, text <> "20") + "21" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 21, text <> "21") + "22" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 22, text <> "22") + "23" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 23, text <> "23") + "24" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 24, text <> "24") + "25" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 25, text <> "25") + "26" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 26, text <> "26") + "27" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 27, text <> "27") + "28" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 28, text <> "28") + "29" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 29, text <> "29") + "30" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 30, text <> "30") + "31" <> src -> + lex_date_end(lex_step(lexer, src), year, month, 31, text <> "31") _ -> Error(IncompleteDate(lexer.position)) } } @@ -351,7 +548,7 @@ fn lex_date_end( let date = calendar.Date(year, month, day) case lexer.src { " " as delimeter <> src | "T" as delimeter <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) let text = text <> delimeter use #(lexer, text, time) <- result.try(lex_time_value(lexer, text)) lex_datetime_offset(lexer, date, time, text) @@ -369,21 +566,21 @@ fn lex_datetime_offset( ) -> Result(#(Lexer, Token), TomlError) { case lexer.src { "Z" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) let text = text <> "Z" let offset = calendar.utc_offset let token = OffsetDateTimeToken(src: text, date:, time:, offset:) Ok(#(lexer, token)) } "+" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) let text = text <> "+" use #(lexer, text, offset) <- result.try(lex_offset(lexer, text, Positive)) let token = OffsetDateTimeToken(src: text, date:, time:, offset:) Ok(#(lexer, token)) } "-" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) let text = text <> "-" use #(lexer, text, offset) <- result.try(lex_offset(lexer, text, Negative)) let token = OffsetDateTimeToken(src: text, date:, time:, offset:) @@ -422,30 +619,30 @@ fn lex_hour_minute( text: String, ) -> Result(#(Lexer, String, Int, Int), TomlError) { use #(lexer, hours, text) <- result.try(case lexer.src { - "00:" as t <> src -> Ok(#(step(lexer, src), 0, text <> t)) - "01:" as t <> src -> Ok(#(step(lexer, src), 1, text <> t)) - "02:" as t <> src -> Ok(#(step(lexer, src), 2, text <> t)) - "03:" as t <> src -> Ok(#(step(lexer, src), 3, text <> t)) - "04:" as t <> src -> Ok(#(step(lexer, src), 4, text <> t)) - "05:" as t <> src -> Ok(#(step(lexer, src), 5, text <> t)) - "06:" as t <> src -> Ok(#(step(lexer, src), 6, text <> t)) - "07:" as t <> src -> Ok(#(step(lexer, src), 7, text <> t)) - "08:" as t <> src -> Ok(#(step(lexer, src), 8, text <> t)) - "09:" as t <> src -> Ok(#(step(lexer, src), 9, text <> t)) - "10:" as t <> src -> Ok(#(step(lexer, src), 10, text <> t)) - "11:" as t <> src -> Ok(#(step(lexer, src), 11, text <> t)) - "12:" as t <> src -> Ok(#(step(lexer, src), 12, text <> t)) - "13:" as t <> src -> Ok(#(step(lexer, src), 13, text <> t)) - "14:" as t <> src -> Ok(#(step(lexer, src), 14, text <> t)) - "15:" as t <> src -> Ok(#(step(lexer, src), 15, text <> t)) - "16:" as t <> src -> Ok(#(step(lexer, src), 16, text <> t)) - "17:" as t <> src -> Ok(#(step(lexer, src), 17, text <> t)) - "18:" as t <> src -> Ok(#(step(lexer, src), 18, text <> t)) - "19:" as t <> src -> Ok(#(step(lexer, src), 19, text <> t)) - "20:" as t <> src -> Ok(#(step(lexer, src), 20, text <> t)) - "21:" as t <> src -> Ok(#(step(lexer, src), 21, text <> t)) - "22:" as t <> src -> Ok(#(step(lexer, src), 22, text <> t)) - "23:" as t <> src -> Ok(#(step(lexer, src), 23, text <> t)) + "00:" as t <> src -> Ok(#(lex_step(lexer, src), 0, text <> t)) + "01:" as t <> src -> Ok(#(lex_step(lexer, src), 1, text <> t)) + "02:" as t <> src -> Ok(#(lex_step(lexer, src), 2, text <> t)) + "03:" as t <> src -> Ok(#(lex_step(lexer, src), 3, text <> t)) + "04:" as t <> src -> Ok(#(lex_step(lexer, src), 4, text <> t)) + "05:" as t <> src -> Ok(#(lex_step(lexer, src), 5, text <> t)) + "06:" as t <> src -> Ok(#(lex_step(lexer, src), 6, text <> t)) + "07:" as t <> src -> Ok(#(lex_step(lexer, src), 7, text <> t)) + "08:" as t <> src -> Ok(#(lex_step(lexer, src), 8, text <> t)) + "09:" as t <> src -> Ok(#(lex_step(lexer, src), 9, text <> t)) + "10:" as t <> src -> Ok(#(lex_step(lexer, src), 10, text <> t)) + "11:" as t <> src -> Ok(#(lex_step(lexer, src), 11, text <> t)) + "12:" as t <> src -> Ok(#(lex_step(lexer, src), 12, text <> t)) + "13:" as t <> src -> Ok(#(lex_step(lexer, src), 13, text <> t)) + "14:" as t <> src -> Ok(#(lex_step(lexer, src), 14, text <> t)) + "15:" as t <> src -> Ok(#(lex_step(lexer, src), 15, text <> t)) + "16:" as t <> src -> Ok(#(lex_step(lexer, src), 16, text <> t)) + "17:" as t <> src -> Ok(#(lex_step(lexer, src), 17, text <> t)) + "18:" as t <> src -> Ok(#(lex_step(lexer, src), 18, text <> t)) + "19:" as t <> src -> Ok(#(lex_step(lexer, src), 19, text <> t)) + "20:" as t <> src -> Ok(#(lex_step(lexer, src), 20, text <> t)) + "21:" as t <> src -> Ok(#(lex_step(lexer, src), 21, text <> t)) + "22:" as t <> src -> Ok(#(lex_step(lexer, src), 22, text <> t)) + "23:" as t <> src -> Ok(#(lex_step(lexer, src), 23, text <> t)) _ -> Error(IncompleteTime(lexer.position)) }) use #(lexer, text, minutes) <- result.try(lex_number_under_60(lexer, text)) @@ -470,11 +667,11 @@ fn lex_seconds( case lexer.src { ":" <> src -> { let text = text <> ":" - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) use #(lexer, text, seconds) <- result.try(lex_number_under_60(lexer, text)) case lexer.src { "." <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) let text = text <> "." parse_time_ns(lexer, text, seconds, 0, 0) } @@ -495,43 +692,43 @@ fn parse_time_ns( ) -> Result(#(Lexer, String, Int, Int), TomlError) { case lexer.src { "0" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "0", seconds, ns * 10 + 0, digits_count + 1) } "1" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "1", seconds, ns * 10 + 1, digits_count + 1) } "2" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "2", seconds, ns * 10 + 2, digits_count + 1) } "3" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "3", seconds, ns * 10 + 3, digits_count + 1) } "4" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "4", seconds, ns * 10 + 4, digits_count + 1) } "5" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "5", seconds, ns * 10 + 5, digits_count + 1) } "6" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "6", seconds, ns * 10 + 6, digits_count + 1) } "7" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "7", seconds, ns * 10 + 7, digits_count + 1) } "8" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "8", seconds, ns * 10 + 8, digits_count + 1) } "9" <> src if digits_count < 9 -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) parse_time_ns(lexer, text <> "9", seconds, ns * 10 + 9, digits_count + 1) } @@ -550,66 +747,66 @@ fn lex_number_under_60( text: String, ) -> Result(#(Lexer, String, Int), TomlError) { case lexer.src { - "00" <> src -> Ok(#(step(lexer, src), text <> "00", 0)) - "01" <> src -> Ok(#(step(lexer, src), text <> "01", 1)) - "02" <> src -> Ok(#(step(lexer, src), text <> "02", 2)) - "03" <> src -> Ok(#(step(lexer, src), text <> "03", 3)) - "04" <> src -> Ok(#(step(lexer, src), text <> "04", 4)) - "05" <> src -> Ok(#(step(lexer, src), text <> "05", 5)) - "06" <> src -> Ok(#(step(lexer, src), text <> "06", 6)) - "07" <> src -> Ok(#(step(lexer, src), text <> "07", 7)) - "08" <> src -> Ok(#(step(lexer, src), text <> "08", 8)) - "09" <> src -> Ok(#(step(lexer, src), text <> "09", 9)) - "10" <> src -> Ok(#(step(lexer, src), text <> "10", 10)) - "11" <> src -> Ok(#(step(lexer, src), text <> "11", 11)) - "12" <> src -> Ok(#(step(lexer, src), text <> "12", 12)) - "13" <> src -> Ok(#(step(lexer, src), text <> "13", 13)) - "14" <> src -> Ok(#(step(lexer, src), text <> "14", 14)) - "15" <> src -> Ok(#(step(lexer, src), text <> "15", 15)) - "16" <> src -> Ok(#(step(lexer, src), text <> "16", 16)) - "17" <> src -> Ok(#(step(lexer, src), text <> "17", 17)) - "18" <> src -> Ok(#(step(lexer, src), text <> "18", 18)) - "19" <> src -> Ok(#(step(lexer, src), text <> "19", 19)) - "20" <> src -> Ok(#(step(lexer, src), text <> "20", 20)) - "21" <> src -> Ok(#(step(lexer, src), text <> "21", 21)) - "22" <> src -> Ok(#(step(lexer, src), text <> "22", 22)) - "23" <> src -> Ok(#(step(lexer, src), text <> "23", 23)) - "24" <> src -> Ok(#(step(lexer, src), text <> "24", 24)) - "25" <> src -> Ok(#(step(lexer, src), text <> "25", 25)) - "26" <> src -> Ok(#(step(lexer, src), text <> "26", 26)) - "27" <> src -> Ok(#(step(lexer, src), text <> "27", 27)) - "28" <> src -> Ok(#(step(lexer, src), text <> "28", 28)) - "29" <> src -> Ok(#(step(lexer, src), text <> "29", 29)) - "30" <> src -> Ok(#(step(lexer, src), text <> "30", 30)) - "31" <> src -> Ok(#(step(lexer, src), text <> "31", 31)) - "32" <> src -> Ok(#(step(lexer, src), text <> "32", 32)) - "33" <> src -> Ok(#(step(lexer, src), text <> "33", 33)) - "34" <> src -> Ok(#(step(lexer, src), text <> "34", 34)) - "35" <> src -> Ok(#(step(lexer, src), text <> "35", 35)) - "36" <> src -> Ok(#(step(lexer, src), text <> "36", 36)) - "37" <> src -> Ok(#(step(lexer, src), text <> "37", 37)) - "38" <> src -> Ok(#(step(lexer, src), text <> "38", 38)) - "39" <> src -> Ok(#(step(lexer, src), text <> "39", 39)) - "40" <> src -> Ok(#(step(lexer, src), text <> "40", 40)) - "41" <> src -> Ok(#(step(lexer, src), text <> "41", 41)) - "42" <> src -> Ok(#(step(lexer, src), text <> "42", 42)) - "43" <> src -> Ok(#(step(lexer, src), text <> "43", 43)) - "44" <> src -> Ok(#(step(lexer, src), text <> "44", 44)) - "45" <> src -> Ok(#(step(lexer, src), text <> "45", 45)) - "46" <> src -> Ok(#(step(lexer, src), text <> "46", 46)) - "47" <> src -> Ok(#(step(lexer, src), text <> "47", 47)) - "48" <> src -> Ok(#(step(lexer, src), text <> "48", 48)) - "49" <> src -> Ok(#(step(lexer, src), text <> "49", 49)) - "50" <> src -> Ok(#(step(lexer, src), text <> "50", 50)) - "51" <> src -> Ok(#(step(lexer, src), text <> "51", 51)) - "52" <> src -> Ok(#(step(lexer, src), text <> "52", 52)) - "53" <> src -> Ok(#(step(lexer, src), text <> "53", 53)) - "54" <> src -> Ok(#(step(lexer, src), text <> "54", 54)) - "55" <> src -> Ok(#(step(lexer, src), text <> "55", 55)) - "56" <> src -> Ok(#(step(lexer, src), text <> "56", 56)) - "57" <> src -> Ok(#(step(lexer, src), text <> "57", 57)) - "58" <> src -> Ok(#(step(lexer, src), text <> "58", 58)) - "59" <> src -> Ok(#(step(lexer, src), text <> "59", 59)) + "00" <> src -> Ok(#(lex_step(lexer, src), text <> "00", 0)) + "01" <> src -> Ok(#(lex_step(lexer, src), text <> "01", 1)) + "02" <> src -> Ok(#(lex_step(lexer, src), text <> "02", 2)) + "03" <> src -> Ok(#(lex_step(lexer, src), text <> "03", 3)) + "04" <> src -> Ok(#(lex_step(lexer, src), text <> "04", 4)) + "05" <> src -> Ok(#(lex_step(lexer, src), text <> "05", 5)) + "06" <> src -> Ok(#(lex_step(lexer, src), text <> "06", 6)) + "07" <> src -> Ok(#(lex_step(lexer, src), text <> "07", 7)) + "08" <> src -> Ok(#(lex_step(lexer, src), text <> "08", 8)) + "09" <> src -> Ok(#(lex_step(lexer, src), text <> "09", 9)) + "10" <> src -> Ok(#(lex_step(lexer, src), text <> "10", 10)) + "11" <> src -> Ok(#(lex_step(lexer, src), text <> "11", 11)) + "12" <> src -> Ok(#(lex_step(lexer, src), text <> "12", 12)) + "13" <> src -> Ok(#(lex_step(lexer, src), text <> "13", 13)) + "14" <> src -> Ok(#(lex_step(lexer, src), text <> "14", 14)) + "15" <> src -> Ok(#(lex_step(lexer, src), text <> "15", 15)) + "16" <> src -> Ok(#(lex_step(lexer, src), text <> "16", 16)) + "17" <> src -> Ok(#(lex_step(lexer, src), text <> "17", 17)) + "18" <> src -> Ok(#(lex_step(lexer, src), text <> "18", 18)) + "19" <> src -> Ok(#(lex_step(lexer, src), text <> "19", 19)) + "20" <> src -> Ok(#(lex_step(lexer, src), text <> "20", 20)) + "21" <> src -> Ok(#(lex_step(lexer, src), text <> "21", 21)) + "22" <> src -> Ok(#(lex_step(lexer, src), text <> "22", 22)) + "23" <> src -> Ok(#(lex_step(lexer, src), text <> "23", 23)) + "24" <> src -> Ok(#(lex_step(lexer, src), text <> "24", 24)) + "25" <> src -> Ok(#(lex_step(lexer, src), text <> "25", 25)) + "26" <> src -> Ok(#(lex_step(lexer, src), text <> "26", 26)) + "27" <> src -> Ok(#(lex_step(lexer, src), text <> "27", 27)) + "28" <> src -> Ok(#(lex_step(lexer, src), text <> "28", 28)) + "29" <> src -> Ok(#(lex_step(lexer, src), text <> "29", 29)) + "30" <> src -> Ok(#(lex_step(lexer, src), text <> "30", 30)) + "31" <> src -> Ok(#(lex_step(lexer, src), text <> "31", 31)) + "32" <> src -> Ok(#(lex_step(lexer, src), text <> "32", 32)) + "33" <> src -> Ok(#(lex_step(lexer, src), text <> "33", 33)) + "34" <> src -> Ok(#(lex_step(lexer, src), text <> "34", 34)) + "35" <> src -> Ok(#(lex_step(lexer, src), text <> "35", 35)) + "36" <> src -> Ok(#(lex_step(lexer, src), text <> "36", 36)) + "37" <> src -> Ok(#(lex_step(lexer, src), text <> "37", 37)) + "38" <> src -> Ok(#(lex_step(lexer, src), text <> "38", 38)) + "39" <> src -> Ok(#(lex_step(lexer, src), text <> "39", 39)) + "40" <> src -> Ok(#(lex_step(lexer, src), text <> "40", 40)) + "41" <> src -> Ok(#(lex_step(lexer, src), text <> "41", 41)) + "42" <> src -> Ok(#(lex_step(lexer, src), text <> "42", 42)) + "43" <> src -> Ok(#(lex_step(lexer, src), text <> "43", 43)) + "44" <> src -> Ok(#(lex_step(lexer, src), text <> "44", 44)) + "45" <> src -> Ok(#(lex_step(lexer, src), text <> "45", 45)) + "46" <> src -> Ok(#(lex_step(lexer, src), text <> "46", 46)) + "47" <> src -> Ok(#(lex_step(lexer, src), text <> "47", 47)) + "48" <> src -> Ok(#(lex_step(lexer, src), text <> "48", 48)) + "49" <> src -> Ok(#(lex_step(lexer, src), text <> "49", 49)) + "50" <> src -> Ok(#(lex_step(lexer, src), text <> "50", 50)) + "51" <> src -> Ok(#(lex_step(lexer, src), text <> "51", 51)) + "52" <> src -> Ok(#(lex_step(lexer, src), text <> "52", 52)) + "53" <> src -> Ok(#(lex_step(lexer, src), text <> "53", 53)) + "54" <> src -> Ok(#(lex_step(lexer, src), text <> "54", 54)) + "55" <> src -> Ok(#(lex_step(lexer, src), text <> "55", 55)) + "56" <> src -> Ok(#(lex_step(lexer, src), text <> "56", 56)) + "57" <> src -> Ok(#(lex_step(lexer, src), text <> "57", 57)) + "58" <> src -> Ok(#(lex_step(lexer, src), text <> "58", 58)) + "59" <> src -> Ok(#(lex_step(lexer, src), text <> "59", 59)) _ -> Error(IncompleteTime(lexer.position)) } } @@ -621,57 +818,58 @@ fn lex_float( text: String, ) -> Result(#(Lexer, Token), TomlError) { case lexer.src { - "_" <> src -> lex_float(step(lexer, src), float, unit, text <> "_") - "0" <> src -> lex_float(step(lexer, src), float, unit *. 0.1, text <> "0") + "_" <> src -> lex_float(lex_step(lexer, src), float, unit, text <> "_") + "0" <> src -> + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "0") "1" <> src -> { let float = float +. 1.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "1") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "1") } "2" <> src -> { let float = float +. 2.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "2") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "2") } "3" <> src -> { let float = float +. 3.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "3") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "3") } "4" <> src -> { let float = float +. 4.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "4") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "4") } "5" <> src -> { let float = float +. 5.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "5") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "5") } "6" <> src -> { let float = float +. 6.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "6") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "6") } "7" <> src -> { let float = float +. 7.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "7") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "7") } "8" <> src -> { let float = float +. 8.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "8") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "8") } "9" <> src -> { let float = float +. 9.0 *. unit - lex_float(step(lexer, src), float, unit *. 0.1, text <> "9") + lex_float(lex_step(lexer, src), float, unit *. 0.1, text <> "9") } "e+" <> src -> - lex_exponent(step(lexer, src), float, text <> "e+", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "e+", 0, Positive) "e-" <> src -> - lex_exponent(step(lexer, src), float, text <> "e-", 0, Negative) + lex_exponent(lex_step(lexer, src), float, text <> "e-", 0, Negative) "e" <> src -> - lex_exponent(step(lexer, src), float, text <> "e", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "e", 0, Positive) "E+" <> src -> - lex_exponent(step(lexer, src), float, text <> "E+", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "E+", 0, Positive) "E-" <> src -> - lex_exponent(step(lexer, src), float, text <> "E-", 0, Negative) + lex_exponent(lex_step(lexer, src), float, text <> "E-", 0, Negative) "E" <> src -> - lex_exponent(step(lexer, src), float, text <> "E", 0, Positive) + lex_exponent(lex_step(lexer, src), float, text <> "E", 0, Positive) _ if unit == 0.1 -> Error(IncompleteFloat(lexer.position - 1)) @@ -694,47 +892,47 @@ fn lex_exponent( ) -> Result(#(Lexer, Token), TomlError) { case lexer.src { "_" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "_", ex, sign) } "0" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "0", ex * 10, sign) } "1" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "1", ex * 10 + 1, sign) } "2" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "2", ex * 10 + 2, sign) } "3" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "3", ex * 10 + 3, sign) } "4" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "4", ex * 10 + 4, sign) } "5" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "5", ex * 10 + 5, sign) } "6" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "6", ex * 10 + 6, sign) } "7" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "7", ex * 10 + 7, sign) } "8" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "8", ex * 10 + 8, sign) } "9" <> src -> { - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) lex_exponent(lexer, n, text <> "9", ex * 10 + 9, sign) } @@ -831,14 +1029,17 @@ fn lex_bare_key( | "8" as c <> src | "9" as c <> src | "-" as c <> src - | "_" as c <> src -> lex_bare_key(step(lexer, src), content <> c) + | "_" as c <> src -> lex_bare_key(lex_step(lexer, src), content <> c) src if content != "" -> lexed(lexer, src, BareKeyToken(content)) _ -> Error(UnknownSequence(lexer.position)) } } -fn lex_comment(src: String, lexer: Lexer) -> Result(#(Lexer, Token), TomlError) { +fn lex_comment( + src: String, + lexer: Lexer, +) -> Result(#(Lexer, Token), TomlError) { case string.split_once(src, "\n") { Ok(#(comment, src)) -> { let token = CommentToken(comment <> "\n") @@ -941,20 +1142,23 @@ fn lex_escape( value: String, ) -> Result(#(Lexer, String, String), TomlError) { case lexer.src { - "x" <> src -> lex_unicode_escape(step(lexer, src), text <> "x", value, 2) - "u" <> src -> lex_unicode_escape(step(lexer, src), text <> "u", value, 4) - "U" <> src -> lex_unicode_escape(step(lexer, src), text <> "U", value, 8) - "t" <> src -> Ok(#(step(lexer, src), text <> "t", value <> "\t")) - "e" <> src -> Ok(#(step(lexer, src), text <> "e", value <> "\u{001b}")) - "b" <> src -> Ok(#(step(lexer, src), text <> "b", value <> "\u{0008}")) - "n" <> src -> Ok(#(step(lexer, src), text <> "n", value <> "\n")) - "r" <> src -> Ok(#(step(lexer, src), text <> "r", value <> "\r")) - "f" <> src -> Ok(#(step(lexer, src), text <> "f", value <> "\f")) - "\"" <> src -> Ok(#(step(lexer, src), text <> "\"", value <> "\"")) - "\\" <> src -> Ok(#(step(lexer, src), text <> "\\", value <> "\\")) + "x" <> src -> + lex_unicode_escape(lex_step(lexer, src), text <> "x", value, 2) + "u" <> src -> + lex_unicode_escape(lex_step(lexer, src), text <> "u", value, 4) + "U" <> src -> + lex_unicode_escape(lex_step(lexer, src), text <> "U", value, 8) + "t" <> src -> Ok(#(lex_step(lexer, src), text <> "t", value <> "\t")) + "e" <> src -> Ok(#(lex_step(lexer, src), text <> "e", value <> "\u{001b}")) + "b" <> src -> Ok(#(lex_step(lexer, src), text <> "b", value <> "\u{0008}")) + "n" <> src -> Ok(#(lex_step(lexer, src), text <> "n", value <> "\n")) + "r" <> src -> Ok(#(lex_step(lexer, src), text <> "r", value <> "\r")) + "f" <> src -> Ok(#(lex_step(lexer, src), text <> "f", value <> "\f")) + "\"" <> src -> Ok(#(lex_step(lexer, src), text <> "\"", value <> "\"")) + "\\" <> src -> Ok(#(lex_step(lexer, src), text <> "\\", value <> "\\")) "\n" <> src -> { let #(whitespace, src) = take_whitespace("\n", src) - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) Ok(#(lexer, text <> whitespace, value)) } _ -> Error(UnknownEscapeSequence(lexer.position)) @@ -980,7 +1184,7 @@ fn lex_unicode_escape( |> result.replace_error(InvalidEscapeSequence(lexer.position)), ) - let lexer = step(lexer, src) + let lexer = lex_step(lexer, src) let text = text <> hex let value = value <> string.from_utf_codepoints([codepoint]) Ok(#(lexer, text, value)) @@ -992,7 +1196,7 @@ fn run_splitter( src: String, ) -> #(Lexer, String, String) { let #(before, split, after) = splitter.split(splitter, src) - let lexer = step(lexer, after) + let lexer = lex_step(lexer, after) #(lexer, before, split) } @@ -1001,7 +1205,7 @@ fn lexed( src: String, token: Token, ) -> Result(#(Lexer, Token), TomlError) { - Ok(#(step(lexer, src), token)) + Ok(#(lex_step(lexer, src), token)) } fn take_whitespace(taken: String, src: String) -> #(String, String) { diff --git a/test/tom/to_symbols_test.gleam b/test/tom/to_symbols_test.gleam new file mode 100644 index 0000000..b18f153 --- /dev/null +++ b/test/tom/to_symbols_test.gleam @@ -0,0 +1,16 @@ +import tom + +pub fn gleam_toml_test() { + let assert Ok(symbols) = + tom.to_symbols( + " +name = \"tom\" +version = \"2.0.2\" + +[dependencies] +gleam_stdlib = \">= 0.33.0 and < 3.0.0\" +", + ) + + assert symbols == [] +} diff --git a/test/tom/tokenise_test.gleam b/test/tom/to_tokens_test.gleam similarity index 82% rename from test/tom/tokenise_test.gleam rename to test/tom/to_tokens_test.gleam index 2ac2ba2..d94e761 100644 --- a/test/tom/tokenise_test.gleam +++ b/test/tom/to_tokens_test.gleam @@ -3,7 +3,7 @@ import gleam/time/calendar import gleam/time/duration import tom.{ BareKeyToken, BasicStringToken, BoolToken, CommaToken, CommentToken, DotToken, - DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFile, EqualsToken, + DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFileToken, EqualsToken, FloatToken, IncompleteDate, IncompleteFloat, IncompleteTime, InfinityToken, IntToken, LeftBraceToken, LeftBracketToken, LiteralStringToken, LocalDateTimeToken, LocalDateToken, LocalTimeToken, MultiLineBasicStringToken, @@ -13,80 +13,83 @@ import tom.{ } pub fn empty_test() { - assert tom.to_tokens("") == Ok([EndOfFile]) + assert tom.to_tokens("") == Ok([EndOfFileToken]) } pub fn equal_test() { - assert tom.to_tokens("=") == Ok([EqualsToken, EndOfFile]) + assert tom.to_tokens("=") == Ok([EqualsToken, EndOfFileToken]) } pub fn dot_test() { - assert tom.to_tokens(".") == Ok([DotToken, EndOfFile]) + assert tom.to_tokens(".") == Ok([DotToken, EndOfFileToken]) } pub fn comma_test() { - assert tom.to_tokens(",") == Ok([CommaToken, EndOfFile]) + assert tom.to_tokens(",") == Ok([CommaToken, EndOfFileToken]) } pub fn braces_test() { - assert tom.to_tokens("{}") == Ok([LeftBraceToken, RightBraceToken, EndOfFile]) + assert tom.to_tokens("{}") + == Ok([LeftBraceToken, RightBraceToken, EndOfFileToken]) } pub fn brackets_test() { assert tom.to_tokens("[]") - == Ok([LeftBracketToken, RightBracketToken, EndOfFile]) + == Ok([LeftBracketToken, RightBracketToken, EndOfFileToken]) } pub fn double_brackets_test() { assert tom.to_tokens("[[]]") - == Ok([DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFile]) + == Ok([DoubleLeftBracketToken, DoubleRightBracketToken, EndOfFileToken]) } pub fn true_test() { - assert tom.to_tokens("true") == Ok([BoolToken(True), EndOfFile]) + assert tom.to_tokens("true") == Ok([BoolToken(True), EndOfFileToken]) } pub fn false_test() { - assert tom.to_tokens("false") == Ok([BoolToken(False), EndOfFile]) + assert tom.to_tokens("false") == Ok([BoolToken(False), EndOfFileToken]) } pub fn nan_test() { - assert tom.to_tokens("nan") == Ok([NanToken(None), EndOfFile]) + assert tom.to_tokens("nan") == Ok([NanToken(None), EndOfFileToken]) } pub fn nan_positive_test() { - assert tom.to_tokens("+nan") == Ok([NanToken(Some(Positive)), EndOfFile]) + assert tom.to_tokens("+nan") == Ok([NanToken(Some(Positive)), EndOfFileToken]) } pub fn nan_negative_test() { - assert tom.to_tokens("-nan") == Ok([NanToken(Some(Negative)), EndOfFile]) + assert tom.to_tokens("-nan") == Ok([NanToken(Some(Negative)), EndOfFileToken]) } pub fn inf_test() { - assert tom.to_tokens("inf") == Ok([InfinityToken(None), EndOfFile]) + assert tom.to_tokens("inf") == Ok([InfinityToken(None), EndOfFileToken]) } pub fn inf_positive_test() { - assert tom.to_tokens("+inf") == Ok([InfinityToken(Some(Positive)), EndOfFile]) + assert tom.to_tokens("+inf") + == Ok([InfinityToken(Some(Positive)), EndOfFileToken]) } pub fn inf_negative_test() { - assert tom.to_tokens("-inf") == Ok([InfinityToken(Some(Negative)), EndOfFile]) + assert tom.to_tokens("-inf") + == Ok([InfinityToken(Some(Negative)), EndOfFileToken]) } pub fn newline_test() { assert tom.to_tokens("\n\n\n") - == Ok([NewlineToken, NewlineToken, NewlineToken, EndOfFile]) + == Ok([NewlineToken, NewlineToken, NewlineToken, EndOfFileToken]) } pub fn comment_test() { assert tom.to_tokens("# Hello, world!\n\n") - == Ok([CommentToken(" Hello, world!\n"), NewlineToken, EndOfFile]) + == Ok([CommentToken(" Hello, world!\n"), NewlineToken, EndOfFileToken]) } pub fn comment_no_newline_test() { assert tom.to_tokens("# Hello, world!") - == Ok([CommentToken(" Hello, world!"), EndOfFile]) + == Ok([CommentToken(" Hello, world!"), EndOfFileToken]) } pub fn spaces_test() { @@ -99,7 +102,7 @@ pub fn spaces_test() { CommentToken(" 1\n"), WhitespaceToken(" "), CommentToken(" 2\n"), - EndOfFile, + EndOfFileToken, ]) } @@ -113,7 +116,7 @@ pub fn tabs_test() { CommentToken(" 1\n"), WhitespaceToken("\t\t\t"), CommentToken(" 2\n"), - EndOfFile, + EndOfFileToken, ]) } @@ -127,7 +130,7 @@ pub fn tabs_and_spaces_test() { CommentToken(" 1\n"), WhitespaceToken("\t \t \t"), CommentToken(" 2\n"), - EndOfFile, + EndOfFileToken, ]) } @@ -135,7 +138,7 @@ pub fn literal_string_test() { assert tom.to_tokens("'Hello'") == Ok([ LiteralStringToken(src: "Hello"), - EndOfFile, + EndOfFileToken, ]) } @@ -157,7 +160,7 @@ pub fn multiline_literal_string_test() { ) == Ok([ MultiLineLiteralStringToken(src: "\n1\n2\n3\n", value: "1\n2\n3\n"), - EndOfFile, + EndOfFileToken, ]) } @@ -165,7 +168,7 @@ pub fn basic_string_test() { assert tom.to_tokens("\"Hello\"") == Ok([ BasicStringToken(src: "Hello", value: "Hello"), - EndOfFile, + EndOfFileToken, ]) } @@ -182,7 +185,7 @@ pub fn basic_string_quote_escape_test() { assert tom.to_tokens("\"\\\"\"") == Ok([ BasicStringToken(src: "\\\"", value: "\""), - EndOfFile, + EndOfFileToken, ]) } @@ -190,7 +193,7 @@ pub fn basic_string_backslash_escape_test() { assert tom.to_tokens("\"\\\\\"") == Ok([ BasicStringToken(src: "\\\\", value: "\\"), - EndOfFile, + EndOfFileToken, ]) } @@ -198,7 +201,7 @@ pub fn basic_string_backspace_escape_test() { assert tom.to_tokens("\"\\b\"") == Ok([ BasicStringToken(src: "\\b", value: "\u{8}"), - EndOfFile, + EndOfFileToken, ]) } @@ -206,7 +209,7 @@ pub fn basic_string_tab_escape_test() { assert tom.to_tokens("\"\\t\"") == Ok([ BasicStringToken(src: "\\t", value: "\t"), - EndOfFile, + EndOfFileToken, ]) } @@ -214,7 +217,7 @@ pub fn basic_string_newline_escape_test() { assert tom.to_tokens("\"\\n\"") == Ok([ BasicStringToken(src: "\\n", value: "\n"), - EndOfFile, + EndOfFileToken, ]) } @@ -222,7 +225,7 @@ pub fn basic_string_form_feed_escape_test() { assert tom.to_tokens("\"\\f\"") == Ok([ BasicStringToken(src: "\\f", value: "\u{c}"), - EndOfFile, + EndOfFileToken, ]) } @@ -230,7 +233,7 @@ pub fn basic_string_carriage_return_escape_test() { assert tom.to_tokens("\"\\r\"") == Ok([ BasicStringToken(src: "\\r", value: "\r"), - EndOfFile, + EndOfFileToken, ]) } @@ -238,7 +241,7 @@ pub fn basic_string_escape_escape_test() { assert tom.to_tokens("\"\\e\"") == Ok([ BasicStringToken(src: "\\e", value: "\u{1b}"), - EndOfFile, + EndOfFileToken, ]) } @@ -246,7 +249,7 @@ pub fn basic_string_hex_escape_test() { assert tom.to_tokens("\"\\x7f\"") == Ok([ BasicStringToken(src: "\\x7f", value: "\u{7f}"), - EndOfFile, + EndOfFileToken, ]) } @@ -254,7 +257,7 @@ pub fn basic_string_hex_escape_upper_range_test() { assert tom.to_tokens("\"\\xff\"") == Ok([ BasicStringToken(src: "\\xff", value: "\u{ff}"), - EndOfFile, + EndOfFileToken, ]) } @@ -262,7 +265,7 @@ pub fn basic_string_unicode_escape_test() { assert tom.to_tokens("\"\\u03B1\"") == Ok([ BasicStringToken(src: "\\u03B1", value: "\u{3b1}"), - EndOfFile, + EndOfFileToken, ]) } @@ -270,7 +273,7 @@ pub fn basic_string_unicode_long_escape_test() { assert tom.to_tokens("\"\\U0001F600\"") == Ok([ BasicStringToken(src: "\\U0001F600", value: "\u{1f600}"), - EndOfFile, + EndOfFileToken, ]) } @@ -278,7 +281,7 @@ pub fn multiline_basic_string_test() { assert tom.to_tokens("\"\"\"\nHello\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\nHello", value: "Hello"), - EndOfFile, + EndOfFileToken, ]) } @@ -286,7 +289,7 @@ pub fn multiline_basic_string_no_newline_test() { assert tom.to_tokens("\"\"\"Hello\"\"\"") == Ok([ MultiLineBasicStringToken(src: "Hello", value: "Hello"), - EndOfFile, + EndOfFileToken, ]) } @@ -294,7 +297,7 @@ pub fn multiline_basic_string_multiline_test() { assert tom.to_tokens("\"\"\"\n1\n2\n3\n\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n1\n2\n3\n", value: "1\n2\n3\n"), - EndOfFile, + EndOfFileToken, ]) } @@ -302,7 +305,7 @@ pub fn multiline_basic_string_newline_test() { assert tom.to_tokens("\"\"\"\n1\n2\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n1\n2", value: "1\n2"), - EndOfFile, + EndOfFileToken, ]) } @@ -315,7 +318,7 @@ pub fn multiline_basic_string_quote_escape_test() { assert tom.to_tokens("\"\"\"\n\\\"\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\\"", value: "\""), - EndOfFile, + EndOfFileToken, ]) } @@ -323,7 +326,7 @@ pub fn multiline_basic_string_backslash_escape_test() { assert tom.to_tokens("\"\"\"\n\\\\\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\\\", value: "\\"), - EndOfFile, + EndOfFileToken, ]) } @@ -331,7 +334,7 @@ pub fn multiline_basic_string_backspace_escape_test() { assert tom.to_tokens("\"\"\"\n\\b\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\b", value: "\u{8}"), - EndOfFile, + EndOfFileToken, ]) } @@ -339,7 +342,7 @@ pub fn multiline_basic_string_tab_escape_test() { assert tom.to_tokens("\"\"\"\n\\t\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\t", value: "\t"), - EndOfFile, + EndOfFileToken, ]) } @@ -347,7 +350,7 @@ pub fn multiline_basic_string_newline_escape_test() { assert tom.to_tokens("\"\"\"\n\\n\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\n", value: "\n"), - EndOfFile, + EndOfFileToken, ]) } @@ -355,7 +358,7 @@ pub fn multiline_basic_string_form_feed_escape_test() { assert tom.to_tokens("\"\"\"\n\\f\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\f", value: "\u{c}"), - EndOfFile, + EndOfFileToken, ]) } @@ -363,7 +366,7 @@ pub fn multiline_basic_string_carriage_return_escape_test() { assert tom.to_tokens("\"\"\"\n\\r\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\r", value: "\r"), - EndOfFile, + EndOfFileToken, ]) } @@ -371,7 +374,7 @@ pub fn multiline_basic_string_escape_escape_test() { assert tom.to_tokens("\"\"\"\n\\e\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\e", value: "\u{1b}"), - EndOfFile, + EndOfFileToken, ]) } @@ -379,7 +382,7 @@ pub fn multiline_basic_string_hex_escape_test() { assert tom.to_tokens("\"\"\"\n\\x7f\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\x7f", value: "\u{7f}"), - EndOfFile, + EndOfFileToken, ]) } @@ -387,7 +390,7 @@ pub fn multiline_basic_string_hex_escape_upper_range_test() { assert tom.to_tokens("\"\"\"\n\\xff\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\xff", value: "\u{ff}"), - EndOfFile, + EndOfFileToken, ]) } @@ -395,7 +398,7 @@ pub fn multiline_basic_string_unicode_escape_test() { assert tom.to_tokens("\"\"\"\n\\u03B1\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\u03B1", value: "\u{3b1}"), - EndOfFile, + EndOfFileToken, ]) } @@ -403,7 +406,7 @@ pub fn multiline_basic_string_unicode_long_escape_test() { assert tom.to_tokens("\"\"\"\n\\U0001F600\"\"\"") == Ok([ MultiLineBasicStringToken(src: "\n\\U0001F600", value: "\u{1f600}"), - EndOfFile, + EndOfFileToken, ]) } @@ -416,7 +419,7 @@ pub fn multiline_basic_string_line_ending_backslash_test() { src: "\nThe quick brown \\\nfox jumps over \\\nthe lazy dog.", value: "The quick brown fox jumps over the lazy dog.", ), - EndOfFile, + EndOfFileToken, ]) } @@ -429,7 +432,7 @@ pub fn multiline_basic_string_line_ending_backslash_spaces_test() { src: "\nThe quick brown \\\n fox jumps over \\\n the lazy dog.", value: "The quick brown fox jumps over the lazy dog.", ), - EndOfFile, + EndOfFileToken, ]) } @@ -442,7 +445,7 @@ pub fn multiline_basic_string_line_ending_backslash_tabs_test() { src: "\nThe quick brown \\\n\tfox jumps over \\\n\tthe lazy dog.", value: "The quick brown fox jumps over the lazy dog.", ), - EndOfFile, + EndOfFileToken, ]) } @@ -451,12 +454,12 @@ pub fn unexpected_test() { } pub fn key_test() { - assert tom.to_tokens("name") == Ok([BareKeyToken("name"), EndOfFile]) + assert tom.to_tokens("name") == Ok([BareKeyToken("name"), EndOfFileToken]) } pub fn key_fancy_test() { assert tom.to_tokens("_H311o-W0rld_") - == Ok([BareKeyToken("_H311o-W0rld_"), EndOfFile]) + == Ok([BareKeyToken("_H311o-W0rld_"), EndOfFileToken]) } pub fn key_value_test() { @@ -468,46 +471,48 @@ pub fn key_value_test() { WhitespaceToken(" "), LeftBracketToken, RightBracketToken, - EndOfFile, + EndOfFileToken, ]) } pub fn number_test() { assert tom.to_tokens("1234567890") - == Ok([IntToken("1234567890", 1_234_567_890), EndOfFile]) + == Ok([IntToken("1234567890", 1_234_567_890), EndOfFileToken]) } pub fn int_positive_test() { assert tom.to_tokens("+1234567890") - == Ok([IntToken("+1234567890", 1_234_567_890), EndOfFile]) + == Ok([IntToken("+1234567890", 1_234_567_890), EndOfFileToken]) } pub fn int_negative_test() { assert tom.to_tokens("-1234567890") - == Ok([IntToken("-1234567890", -1_234_567_890), EndOfFile]) + == Ok([IntToken("-1234567890", -1_234_567_890), EndOfFileToken]) } pub fn int_underscore_test() { assert tom.to_tokens("12_345_67__890") - == Ok([IntToken("12_345_67__890", 1_234_567_890), EndOfFile]) + == Ok([IntToken("12_345_67__890", 1_234_567_890), EndOfFileToken]) } pub fn float_test() { - assert tom.to_tokens("12.34") == Ok([FloatToken("12.34", 12.34), EndOfFile]) + assert tom.to_tokens("12.34") + == Ok([FloatToken("12.34", 12.34), EndOfFileToken]) } pub fn float_negative_test() { assert tom.to_tokens("-12.34") - == Ok([FloatToken("-12.34", -12.34), EndOfFile]) + == Ok([FloatToken("-12.34", -12.34), EndOfFileToken]) } pub fn float_positive_test() { - assert tom.to_tokens("+12.34") == Ok([FloatToken("+12.34", 12.34), EndOfFile]) + assert tom.to_tokens("+12.34") + == Ok([FloatToken("+12.34", 12.34), EndOfFileToken]) } pub fn float_underscore_test() { assert tom.to_tokens("12_34.5_67__890") - == Ok([FloatToken("12_34.5_67__890", 1234.567_89), EndOfFile]) + == Ok([FloatToken("12_34.5_67__890", 1234.567_89), EndOfFileToken]) } pub fn float_incomplete_test() { @@ -515,38 +520,38 @@ pub fn float_incomplete_test() { } pub fn lex_float_exponent_test() { - assert tom.to_tokens("1e6") == Ok([FloatToken("1e6", 1.0e6), EndOfFile]) + assert tom.to_tokens("1e6") == Ok([FloatToken("1e6", 1.0e6), EndOfFileToken]) } pub fn lex_float_exponent_uppercase_test() { - assert tom.to_tokens("1E6") == Ok([FloatToken("1E6", 1.0e6), EndOfFile]) + assert tom.to_tokens("1E6") == Ok([FloatToken("1E6", 1.0e6), EndOfFileToken]) } pub fn lex_float_exponent_negative_test() { assert tom.to_tokens("-2e-22") - == Ok([FloatToken("-2e-22", -2.0e-22), EndOfFile]) + == Ok([FloatToken("-2e-22", -2.0e-22), EndOfFileToken]) } pub fn lex_float_decimal_and_exponent_test() { assert tom.to_tokens("6.626e25") - == Ok([FloatToken("6.626e25", 6.626e25), EndOfFile]) + == Ok([FloatToken("6.626e25", 6.626e25), EndOfFileToken]) } pub fn lex_float_decimal_and_exponent_positive_test() { assert tom.to_tokens("6.626e+25") - == Ok([FloatToken("6.626e+25", 6.626e25), EndOfFile]) + == Ok([FloatToken("6.626e+25", 6.626e25), EndOfFileToken]) } pub fn lex_float_decimal_and_exponent_negative_test() { assert tom.to_tokens("6.626e-25") - == Ok([FloatToken("6.626e-25", 6.626e-25), EndOfFile]) + == Ok([FloatToken("6.626e-25", 6.626e-25), EndOfFileToken]) } pub fn local_time_test() { assert tom.to_tokens("07:32:00") == Ok([ LocalTimeToken("07:32:00", calendar.TimeOfDay(7, 32, 0, 0)), - EndOfFile, + EndOfFileToken, ]) } @@ -554,7 +559,7 @@ pub fn local_time_fractional_seconds_test() { assert tom.to_tokens("00:32:00.1234") == Ok([ LocalTimeToken("00:32:00.1234", calendar.TimeOfDay(0, 32, 0, 123_400_000)), - EndOfFile, + EndOfFileToken, ]) } @@ -566,7 +571,7 @@ pub fn local_time_incomplete_seconds_test() { assert tom.to_tokens("07:32") == Ok([ LocalTimeToken("07:32", calendar.TimeOfDay(7, 32, 0, 0)), - EndOfFile, + EndOfFileToken, ]) } @@ -578,7 +583,7 @@ pub fn local_date_test() { assert tom.to_tokens("1991-01-05") == Ok([ LocalDateToken("1991-01-05", calendar.Date(1991, calendar.January, 5)), - EndOfFile, + EndOfFileToken, ]) } @@ -598,7 +603,7 @@ pub fn local_date_time_test() { calendar.Date(1991, calendar.January, 5), calendar.TimeOfDay(7, 32, 0, 0), ), - EndOfFile, + EndOfFileToken, ]) } @@ -611,7 +616,7 @@ pub fn offset_date_time_test() { calendar.TimeOfDay(7, 32, 0, 0), calendar.utc_offset, ), - EndOfFile, + EndOfFileToken, ]) } @@ -624,7 +629,7 @@ pub fn offset_date_time_positive_offset_test() { calendar.TimeOfDay(7, 32, 0, 0), duration.add(duration.hours(7), duration.minutes(30)), ), - EndOfFile, + EndOfFileToken, ]) } @@ -637,6 +642,6 @@ pub fn offset_date_time_negative_offset_test() { calendar.TimeOfDay(7, 32, 0, 0), duration.add(duration.hours(-5), duration.minutes(-15)), ), - EndOfFile, + EndOfFileToken, ]) }