From 0803912c7fe5f034914e3c63347d5b6a4d2f23c8 Mon Sep 17 00:00:00 2001 From: Vincent LE GOFF Date: Thu, 23 May 2019 20:48:54 +0200 Subject: TOML: Move to encoding dir (denoland/deno_std#435) Original: https://github.com/denoland/deno_std/commit/7a722ceffc0b60305863c1417ef22b835db0d58d --- encoding/README.md | 118 ++++++++ encoding/test.ts | 2 + encoding/testdata/CRLF.toml | 3 + encoding/testdata/arrayTable.toml | 12 + encoding/testdata/arrays.toml | 8 + encoding/testdata/boolean.toml | 3 + encoding/testdata/cargo.toml | 56 ++++ encoding/testdata/cargoTest.toml | 147 ++++++++++ encoding/testdata/datetime.toml | 8 + encoding/testdata/float.toml | 23 ++ encoding/testdata/inlineTable.toml | 7 + encoding/testdata/integer.toml | 20 ++ encoding/testdata/simple.toml | 5 + encoding/testdata/string.toml | 30 +++ encoding/testdata/table.toml | 13 + encoding/toml.ts | 538 +++++++++++++++++++++++++++++++++++++ encoding/toml_test.ts | 389 +++++++++++++++++++++++++++ 17 files changed, 1382 insertions(+) create mode 100644 encoding/README.md create mode 100644 encoding/test.ts create mode 100644 encoding/testdata/CRLF.toml create mode 100644 encoding/testdata/arrayTable.toml create mode 100644 encoding/testdata/arrays.toml create mode 100644 encoding/testdata/boolean.toml create mode 100644 encoding/testdata/cargo.toml create mode 100644 encoding/testdata/cargoTest.toml create mode 100644 encoding/testdata/datetime.toml create mode 100644 encoding/testdata/float.toml create mode 100644 encoding/testdata/inlineTable.toml create mode 100644 encoding/testdata/integer.toml create mode 100644 encoding/testdata/simple.toml create mode 100644 encoding/testdata/string.toml create mode 100644 encoding/testdata/table.toml create mode 100644 encoding/toml.ts create mode 100644 encoding/toml_test.ts (limited to 'encoding') diff --git a/encoding/README.md b/encoding/README.md new file mode 100644 index 000000000..e30d972f3 --- /dev/null +++ b/encoding/README.md @@ -0,0 +1,118 @@ +# TOML + +This module parse TOML files. It follows as much as possible the +[TOML specs](https://github.com/toml-lang/toml). Be sure to read the supported +types as not every specs is supported at the moment and the handling in +TypeScript side is a bit different. + +## Supported types and handling + +- :heavy_check_mark: [Keys](https://github.com/toml-lang/toml#string) +- :exclamation: [String](https://github.com/toml-lang/toml#string) +- :heavy_check_mark: + [Multiline String](https://github.com/toml-lang/toml#string) +- :heavy_check_mark: [Literal String](https://github.com/toml-lang/toml#string) +- :exclamation: [Integer](https://github.com/toml-lang/toml#integer) +- :heavy_check_mark: [Float](https://github.com/toml-lang/toml#float) +- :heavy_check_mark: [Boolean](https://github.com/toml-lang/toml#boolean) +- :heavy_check_mark: + [Offset Date-time](https://github.com/toml-lang/toml#offset-date-time) +- :heavy_check_mark: + [Local Date-time](https://github.com/toml-lang/toml#local-date-time) +- :heavy_check_mark: [Local Date](https://github.com/toml-lang/toml#local-date) +- :exclamation: [Local Time](https://github.com/toml-lang/toml#local-time) +- :heavy_check_mark: [Table](https://github.com/toml-lang/toml#table) +- :heavy_check_mark: [Inline Table](https://github.com/toml-lang/toml#inline-table) +- :exclamation: [Array of Tables](https://github.com/toml-lang/toml#array-of-tables) + +:exclamation: _Supported with warnings see [Warning](#Warning)._ + +### :warning: Warning + +#### String + +- Regex : Due to the spec, there is no flag to detect regex properly + in a TOML declaration. So the regex is stored as string. + +#### Integer + +For **Binary** / **Octal** / **Hexadecimal** numbers, +they are stored as string to be not interpreted as Decimal. + +#### Local Time + +Because local time does not exist in JavaScript, the local time is stored as a string. + +#### Inline Table + +Inline tables are supported. See below: + +```toml +animal = { type = { name = "pug" } } +# Output +animal = { type.name = "pug" } +# Output { animal : { type : { name : "pug" } } +animal.as.leaders = "tosin" +# Output { animal: { as: { leaders: "tosin" } } } +"tosin.abasi" = "guitarist" +# Output +"tosin.abasi" : "guitarist" +``` + +#### Array of Tables + +At the moment only simple declarations like below are supported: + +```toml +[[bin]] +name = "deno" +path = "cli/main.rs" + +[[bin]] +name = "deno_core" +path = "src/foo.rs" + +[[nib]] +name = "node" +path = "not_found" +``` + +will output: + +```json +{ + "bin": [ + { "name": "deno", "path": "cli/main.rs" }, + { "name": "deno_core", "path": "src/foo.rs" } + ], + "nib": [{ "name": "node", "path": "not_found" }] +} +``` + +## Usage + +### Parse + +```ts +import { parse } from "./parser.ts"; +import { readFileStrSync } from "../fs/read_file_str.ts"; + +const tomlObject = parse(readFileStrSync("file.toml")); + +const tomlString = 'foo.bar = "Deno"'; +const tomlObject22 = parse(tomlString); +``` + +### Stringify + +```ts +import { stringify } from "./parser.ts"; +const obj = { + bin: [ + { name: "deno", path: "cli/main.rs" }, + { name: "deno_core", path: "src/foo.rs" } + ], + nib: [{ name: "node", path: "not_found" }] +}; +const tomlString = stringify(obj); +``` diff --git a/encoding/test.ts b/encoding/test.ts new file mode 100644 index 000000000..4ee03572d --- /dev/null +++ b/encoding/test.ts @@ -0,0 +1,2 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +import "./toml_test.ts"; diff --git a/encoding/testdata/CRLF.toml b/encoding/testdata/CRLF.toml new file mode 100644 index 000000000..92264888a --- /dev/null +++ b/encoding/testdata/CRLF.toml @@ -0,0 +1,3 @@ +[boolean] +bool1 = true +bool2 = false \ No newline at end of file diff --git a/encoding/testdata/arrayTable.toml b/encoding/testdata/arrayTable.toml new file mode 100644 index 000000000..3788b7e7c --- /dev/null +++ b/encoding/testdata/arrayTable.toml @@ -0,0 +1,12 @@ + +[[bin]] +name = "deno" +path = "cli/main.rs" + +[[bin]] +name = "deno_core" +path = "src/foo.rs" + +[[nib]] +name = "node" +path = "not_found" \ No newline at end of file diff --git a/encoding/testdata/arrays.toml b/encoding/testdata/arrays.toml new file mode 100644 index 000000000..5d5913d0c --- /dev/null +++ b/encoding/testdata/arrays.toml @@ -0,0 +1,8 @@ +[arrays] +data = [ ["gamma", "delta"], [1, 2] ] + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] diff --git a/encoding/testdata/boolean.toml b/encoding/testdata/boolean.toml new file mode 100644 index 000000000..242d29c96 --- /dev/null +++ b/encoding/testdata/boolean.toml @@ -0,0 +1,3 @@ +[boolean] # i hate comments +bool1 = true +bool2 = false \ No newline at end of file diff --git a/encoding/testdata/cargo.toml b/encoding/testdata/cargo.toml new file mode 100644 index 000000000..291aa7db6 --- /dev/null +++ b/encoding/testdata/cargo.toml @@ -0,0 +1,56 @@ +# Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +# Dummy package info required by `cargo fetch`. +# Use tools/sync_third_party.py to install deps after editing this file. +# Deno does not build with cargo. Deno uses a build system called gn. +# See build_extra/rust/BUILD.gn for the manually built configuration of rust +# crates. + +[workspace] +members = [ + "./", + "core", +] + +[[bin]] +name = "deno" +path = "cli/main.rs" + +[package] +name = "deno" +version = "0.3.4" +edition = "2018" + +[dependencies] +deno_core = { path = "./core" } + +ansi_term = "0.11.0" +atty = "0.2.11" +dirs = "1.0.5" +flatbuffers = "0.5.0" +futures = "0.1.25" +getopts = "0.2.18" +http = "0.1.16" +hyper = "0.12.24" +hyper-rustls = "0.16.0" +integer-atomics = "1.0.2" +lazy_static = "1.3.0" +libc = "0.2.49" +log = "0.4.6" +rand = "0.6.5" +regex = "1.1.0" +remove_dir_all = "0.5.1" +ring = "0.14.6" +rustyline = "3.0.0" +serde_json = "1.0.38" +source-map-mappings = "0.5.0" +tempfile = "3.0.7" +tokio = "0.1.15" +tokio-executor = "0.1.6" +tokio-fs = "0.1.5" +tokio-io = "0.1.11" +tokio-process = "0.2.3" +tokio-threadpool = "0.1.11" +url = "1.7.2" + +[target.'cfg(windows)'.dependencies] +winapi = "0.3.6" diff --git a/encoding/testdata/cargoTest.toml b/encoding/testdata/cargoTest.toml new file mode 100644 index 000000000..47e7f6e4d --- /dev/null +++ b/encoding/testdata/cargoTest.toml @@ -0,0 +1,147 @@ +# This is a TOML document. + +title = "TOML Example" + +[deeply.nested.object.in.the.toml] +name = "Tom Preston-Werner" +dob = 2009-05-27T07:32:00 + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # Indentation (tabs and/or spaces) is allowed but not required + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] + +[strings] +str0 = "deno" +str1 = """ +Roses are red + Violets are blue""" +# On a Unix system, the above multi-line string will most likely be the same as: +str2 = "Roses are red\nViolets are blue" + +# On a Windows system, it will most likely be equivalent to: +str3 = "Roses are red\r\nViolets are blue" +str4 = "The quick brown fox jumps over the lazy dog." +str5 = "this is a \"quote\"" + +str5 = """ +The quick brown \ + + + fox jumps over \ + the lazy dog.""" + +str6 = """\ + The quick brown \ + fox jumps over \ + the lazy dog.\ + """ +lines = ''' +The first newline is +trimmed in raw strings. + All other whitespace + is preserved. +''' + +[Integer] +int1 = +99 +int2 = 42 +int3 = 0 +int4 = -17 +int5 = 1_000 +int6 = 5_349_221 +int7 = 1_2_3_4_5 # VALID but discouraged + +# hexadecimal with prefix `0x` +hex1 = 0xDEADBEEF +hex2 = 0xdeadbeef +hex3 = 0xdead_beef + +# octal with prefix `0o` +oct1 = 0o01234567 +oct2 = 0o755 # useful for Unix file permissions + +# binary with prefix `0b` +bin1 = 0b11010110 + +[Date-Time] +odt1 = 1979-05-27T07:32:00Z +odt2 = 1979-05-27T00:32:00-07:00 +odt3 = 1979-05-27T00:32:00.999999-07:00 +odt4 = 1979-05-27 07:32:00Z +ld1 = 1979-05-27 +lt1 = 07:32:00 #buggy +lt2 = 00:32:00.999999 #buggy + +[boolean] +bool1 = true +bool2 = false + +[float] +# fractional +flt1 = +1.0 +flt2 = 3.1415 +flt3 = -0.01 + +# exponent +flt4 = 5e+22 +flt5 = 1e6 +flt6 = -2E-2 + +# both +flt7 = 6.626e-34 +flt8 = 224_617.445_991_228 +# infinity +sf1 = inf # positive infinity +sf2 = +inf # positive infinity +sf3 = -inf # negative infinity + +# not a number +sf4 = nan # actual sNaN/qNaN encoding is implementation specific +sf5 = +nan # same as `nan` +sf6 = -nan # valid, actual encoding is implementation specific + +[Table] +name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 } +animal = { type.name = "pug" } + +[[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + +[[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" diff --git a/encoding/testdata/datetime.toml b/encoding/testdata/datetime.toml new file mode 100644 index 000000000..62377a4ba --- /dev/null +++ b/encoding/testdata/datetime.toml @@ -0,0 +1,8 @@ +[datetime] +odt1 = 1979-05-27T07:32:00Z # Comment +odt2 = 1979-05-27T00:32:00-07:00 # Comment +odt3 = 1979-05-27T00:32:00.999999-07:00 # Comment +odt4 = 1979-05-27 07:32:00Z # Comment +ld1 = 1979-05-27 # Comment +lt1 = 07:32:00 # Comment +lt2 = 00:32:00.999999 # Comment diff --git a/encoding/testdata/float.toml b/encoding/testdata/float.toml new file mode 100644 index 000000000..6a384179c --- /dev/null +++ b/encoding/testdata/float.toml @@ -0,0 +1,23 @@ +[float] +# fractional +flt1 = +1.0 # Comment +flt2 = 3.1415 # Comment +flt3 = -0.01 # Comment + +# exponent +flt4 = 5e+22 # Comment +flt5 = 1e6 # Comment +flt6 = -2E-2 # Comment + +# both +flt7 = 6.626e-34 # Comment +flt8 = 224_617.445_991_228 # Comment +# infinity +sf1 = inf # positive infinity +sf2 = +inf # positive infinity +sf3 = -inf # negative infinity + +# not a number +sf4 = nan # actual sNaN/qNaN encoding is implementation specific +sf5 = +nan # same as `nan` +sf6 = -nan # valid, actual encoding is implementation specific \ No newline at end of file diff --git a/encoding/testdata/inlineTable.toml b/encoding/testdata/inlineTable.toml new file mode 100644 index 000000000..203cb16db --- /dev/null +++ b/encoding/testdata/inlineTable.toml @@ -0,0 +1,7 @@ +[inlinetable] +name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 } +dog = { type = { name = "pug" } } +animal.as.leaders = "tosin" +"tosin.abasi" = "guitarist" +nile = { derek.roddy = "drummer", also = { malevolant.creation = { drum.kit = "Tama" } } } \ No newline at end of file diff --git a/encoding/testdata/integer.toml b/encoding/testdata/integer.toml new file mode 100644 index 000000000..3bd781e8f --- /dev/null +++ b/encoding/testdata/integer.toml @@ -0,0 +1,20 @@ +[integer] +int1 = +99 +int2 = 42 +int3 = 0 +int4 = -17 +int5 = 1_000 +int6 = 5_349_221 +int7 = 1_2_3_4_5 # VALID but discouraged + +# hexadecimal with prefix `0x` +hex1 = 0xDEADBEEF +hex2 = 0xdeadbeef +hex3 = 0xdead_beef + +# octal with prefix `0o` +oct1 = 0o01234567 +oct2 = 0o755 # useful for Unix file permissions + +# binary with prefix `0b` +bin1 = 0b11010110 \ No newline at end of file diff --git a/encoding/testdata/simple.toml b/encoding/testdata/simple.toml new file mode 100644 index 000000000..f3f6c1036 --- /dev/null +++ b/encoding/testdata/simple.toml @@ -0,0 +1,5 @@ +deno = "is" +not = "[node]" +regex = '<\i\c*\s*>' +NANI = '何?!' +comment = "Comment inside # the comment" # Comment diff --git a/encoding/testdata/string.toml b/encoding/testdata/string.toml new file mode 100644 index 000000000..f811824eb --- /dev/null +++ b/encoding/testdata/string.toml @@ -0,0 +1,30 @@ +[strings] +str0 = "deno" +str1 = """ +Roses are not Deno + Violets are not Deno either""" +# On a Unix system, the above multi-line string will most likely be the same as: +str2 = "Roses are not Deno\nViolets are not Deno either" + +# On a Windows system, it will most likely be equivalent to: +str3 = "Roses are not Deno\r\nViolets are not Deno either" +str4 = "this is a \"quote\"" + +str5 = """ +The quick brown \ + + + fox jumps over \ + the lazy dog.""" + +str6 = """\ + The quick brown \ + fox jumps over \ + the lazy dog.\ + """ +lines = ''' +The first newline is +trimmed in raw strings. + All other whitespace + is preserved. +''' \ No newline at end of file diff --git a/encoding/testdata/table.toml b/encoding/testdata/table.toml new file mode 100644 index 000000000..7008e6fb0 --- /dev/null +++ b/encoding/testdata/table.toml @@ -0,0 +1,13 @@ +[deeply.nested.object.in.the.toml] +name = "Tom Preston-Werner" + +[servers] + + # Indentation (tabs and/or spaces) is allowed but not required + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc20" \ No newline at end of file diff --git a/encoding/toml.ts b/encoding/toml.ts new file mode 100644 index 000000000..cc96322fb --- /dev/null +++ b/encoding/toml.ts @@ -0,0 +1,538 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +import { deepAssign } from "../util/deep_assign.ts"; +import { pad } from "../strings/pad.ts"; + +class KeyValuePair { + key: string; + value: unknown; +} + +class ParserGroup { + type: string; + name: string; + arrValues: unknown[] = []; + objValues: object = {}; +} + +class ParserContext { + currentGroup?: ParserGroup; + output: object = {}; +} + +class Parser { + tomlLines: string[]; + context: ParserContext; + constructor(tomlString: string) { + this.tomlLines = this._split(tomlString); + this.context = new ParserContext(); + } + _sanitize(): void { + const out = []; + for (let i = 0; i < this.tomlLines.length; i++) { + const s = this.tomlLines[i]; + const trimmed = s.trim(); + if (trimmed !== "" && trimmed[0] !== "#") { + out.push(s); + } + } + this.tomlLines = out; + this._mergeMultilines(); + } + + _mergeMultilines(): void { + function arrayStart(line: string): boolean { + const reg = /.*=\s*\[/g; + return reg.test(line) && !(line[line.length - 1] === "]"); + } + + function arrayEnd(line: string): boolean { + return line[line.length - 1] === "]"; + } + + function stringStart(line: string): boolean { + const m = line.match(/.*=\s*(?:\"\"\"|''')/); + if (!m) { + return false; + } + return !line.endsWith(`"""`) || !line.endsWith(`'''`); + } + + function stringEnd(line: string): boolean { + return line.endsWith(`'''`) || line.endsWith(`"""`); + } + + function isLiteralString(line: string): boolean { + return line.match(/'''/) ? true : false; + } + + let merged = [], + acc = [], + isLiteral = false, + capture = false, + captureType = "", + merge = false; + + for (let i = 0; i < this.tomlLines.length; i++) { + const line = this.tomlLines[i]; + const trimmed = line.trim(); + if (!capture && arrayStart(trimmed)) { + capture = true; + captureType = "array"; + } else if (!capture && stringStart(trimmed)) { + isLiteral = isLiteralString(trimmed); + capture = true; + captureType = "string"; + } else if (capture && arrayEnd(trimmed)) { + merge = true; + } else if (capture && stringEnd(trimmed)) { + merge = true; + } + + if (capture) { + if (isLiteral) { + acc.push(line); + } else { + acc.push(trimmed); + } + } else { + if (isLiteral) { + merged.push(line); + } else { + merged.push(trimmed); + } + } + + if (merge) { + capture = false; + merge = false; + if (captureType === "string") { + merged.push( + acc + .join("\n") + .replace(/"""/g, '"') + .replace(/'''/g, `'`) + .replace(/\n/g, "\\n") + ); + isLiteral = false; + } else { + merged.push(acc.join("")); + } + captureType = ""; + acc = []; + } + } + this.tomlLines = merged; + } + _unflat(keys: string[], values: object = {}, cObj: object = {}): object { + let out = {}; + if (keys.length === 0) { + return cObj; + } else { + if (Object.keys(cObj).length === 0) { + cObj = values; + } + let key = keys.pop(); + out[key] = cObj; + return this._unflat(keys, values, out); + } + } + _groupToOutput(): void { + const arrProperty = this.context.currentGroup.name + .replace(/"/g, "") + .replace(/'/g, "") + .split("."); + let u = {}; + if (this.context.currentGroup.type === "array") { + u = this._unflat(arrProperty, this.context.currentGroup.arrValues); + } else { + u = this._unflat(arrProperty, this.context.currentGroup.objValues); + } + deepAssign(this.context.output, u); + delete this.context.currentGroup; + } + _split(str: string): string[] { + let out = []; + out.push(...str.split("\n")); + return out; + } + _isGroup(line: string): boolean { + const t = line.trim(); + return t[0] === "[" && /\[(.*)\]/.exec(t) ? true : false; + } + _isDeclaration(line: string): boolean { + return line.split("=").length > 1; + } + _createGroup(line: string): void { + const captureReg = /\[(.*)\]/; + if (this.context.currentGroup) { + this._groupToOutput(); + } + let g = new ParserGroup(); + g.name = line.match(captureReg)[1]; + if (g.name.match(/\[.*\]/)) { + g.type = "array"; + g.name = g.name.match(captureReg)[1]; + } else { + g.type = "object"; + } + this.context.currentGroup = g; + } + _processDeclaration(line: string): KeyValuePair { + let kv = new KeyValuePair(); + const idx = line.indexOf("="); + kv.key = line.substring(0, idx).trim(); + kv.value = this._parseData(line.slice(idx + 1)); + return kv; + } + // TODO (zekth) Need refactor using ACC + _parseData(dataString: string): unknown { + dataString = dataString.trim(); + + if (this._isDate(dataString)) { + return new Date(dataString.split("#")[0].trim()); + } + + if (this._isLocalTime(dataString)) { + return eval(`"${dataString.split("#")[0].trim()}"`); + } + + const cut3 = dataString.substring(0, 3).toLowerCase(); + const cut4 = dataString.substring(0, 4).toLowerCase(); + if (cut3 === "inf" || cut4 === "+inf") { + return Infinity; + } + if (cut4 === "-inf") { + return -Infinity; + } + + if (cut3 === "nan" || cut4 === "+nan" || cut4 === "-nan") { + return NaN; + } + + // If binary / octal / hex + const hex = /(0(?:x|o|b)[0-9a-f_]*)[^#]/gi.exec(dataString); + if (hex && hex[0]) { + return hex[0].trim(); + } + + const testNumber = this._isParsableNumber(dataString); + if (testNumber && !isNaN(testNumber as number)) { + return testNumber; + } + + const invalidArr = /,\]/g.exec(dataString); + if (invalidArr) { + dataString = dataString.replace(/,]/g, "]"); + } + const m = /(?:\'|\[|{|\").*(?:\'|\]|\"|})\s*[^#]/g.exec(dataString); + if (m) { + dataString = m[0].trim(); + } + if (dataString[0] === "{" && dataString[dataString.length - 1] === "}") { + const reg = /([a-zA-Z0-9-_\.]*) (=)/gi; + let result; + while ((result = reg.exec(dataString))) { + let ogVal = result[0]; + let newVal = ogVal + .replace(result[1], `"${result[1]}"`) + .replace(result[2], ":"); + dataString = dataString.replace(ogVal, newVal); + } + return JSON.parse(dataString); + } + + // Handle First and last EOL for multiline strings + if (dataString.startsWith(`"\\n`)) { + dataString = dataString.replace(`"\\n`, `"`); + } else if (dataString.startsWith(`'\\n`)) { + dataString = dataString.replace(`'\\n`, `'`); + } + if (dataString.endsWith(`\\n"`)) { + dataString = dataString.replace(`\\n"`, `"`); + } else if (dataString.endsWith(`\\n'`)) { + dataString = dataString.replace(`\\n'`, `'`); + } + return eval(dataString); + } + _isLocalTime(str: string): boolean { + const reg = /(\d{2}):(\d{2}):(\d{2})/; + return reg.test(str); + } + _isParsableNumber(dataString: string): number | boolean { + const m = /((?:\+|-|)[0-9_\.e+\-]*)[^#]/i.exec(dataString.trim()); + if (!m) { + return false; + } else { + return parseFloat(m[0].replace(/_/g, "")); + } + } + _isDate(dateStr: string): boolean { + const reg = /\d{4}-\d{2}-\d{2}/; + return reg.test(dateStr); + } + _parseDeclarationName(declaration: string): string[] { + const out = []; + let acc = []; + let inLiteral = false; + for (let i = 0; i < declaration.length; i++) { + const c = declaration[i]; + switch (c) { + case ".": + if (!inLiteral) { + out.push(acc.join("")); + acc = []; + } else { + acc.push(c); + } + break; + case `"`: + if (inLiteral) { + inLiteral = false; + } else { + inLiteral = true; + } + break; + default: + acc.push(c); + break; + } + } + if (acc.length !== 0) { + out.push(acc.join("")); + } + return out; + } + _parseLines(): void { + for (let i = 0; i < this.tomlLines.length; i++) { + const line = this.tomlLines[i]; + + // TODO (zekth) Handle unflat of array of tables + if (this._isGroup(line)) { + // if the current group is an array we push the + // parsed objects in it. + if ( + this.context.currentGroup && + this.context.currentGroup.type === "array" + ) { + this.context.currentGroup.arrValues.push( + this.context.currentGroup.objValues + ); + this.context.currentGroup.objValues = {}; + } + // If we need to create a group or to change group + if ( + !this.context.currentGroup || + (this.context.currentGroup && + this.context.currentGroup.name !== + line.replace(/\[/g, "").replace(/\]/g, "")) + ) { + this._createGroup(line); + continue; + } + } + if (this._isDeclaration(line)) { + let kv = this._processDeclaration(line); + let key = kv.key; + let value = kv.value; + if (!this.context.currentGroup) { + this.context.output[key] = value; + } else { + this.context.currentGroup.objValues[key] = value; + } + } + } + if (this.context.currentGroup) { + if (this.context.currentGroup.type === "array") { + this.context.currentGroup.arrValues.push( + this.context.currentGroup.objValues + ); + } + this._groupToOutput(); + } + } + _cleanOutput(): void { + this._propertyClean(this.context.output); + } + _propertyClean(obj: object): void { + const keys = Object.keys(obj); + for (let i = 0; i < keys.length; i++) { + let k = keys[i]; + let v = obj[k]; + let pathDeclaration = this._parseDeclarationName(k); + delete obj[k]; + if (pathDeclaration.length > 1) { + k = pathDeclaration.shift(); + k = k.replace(/"/g, ""); + v = this._unflat(pathDeclaration, v as object); + } else { + k = k.replace(/"/g, ""); + } + obj[k] = v; + if (v instanceof Object) { + this._propertyClean(v); + } + } + } + parse(): object { + this._sanitize(); + this._parseLines(); + this._cleanOutput(); + return this.context.output; + } +} + +class Dumper { + maxPad: number = 0; + srcObject: object; + output: string[] = []; + constructor(srcObjc: object) { + this.srcObject = srcObjc; + } + dump(): string[] { + this.output = this._parse(this.srcObject); + this.output = this._format(); + return this.output; + } + _parse(obj: object, path: string = ""): string[] { + const out = []; + const props = Object.keys(obj); + const propObj = props.filter( + (e): boolean => + (obj[e] instanceof Array && !this._isSimplySerializable(obj[e][0])) || + !this._isSimplySerializable(obj[e]) + ); + const propPrim = props.filter( + (e): boolean => + !(obj[e] instanceof Array && !this._isSimplySerializable(obj[e][0])) && + this._isSimplySerializable(obj[e]) + ); + const k = propPrim.concat(propObj); + for (let i = 0; i < k.length; i++) { + const prop = k[i]; + const value = obj[prop]; + if (value instanceof Date) { + out.push(this._dateDeclaration(prop, value)); + } else if (typeof value === "string" || value instanceof RegExp) { + out.push(this._strDeclaration(prop, value.toString())); + } else if (typeof value === "number") { + out.push(this._numberDeclaration(prop, value)); + } else if ( + value instanceof Array && + this._isSimplySerializable(value[0]) + ) { + // only if primitives types in the array + out.push(this._arrayDeclaration(prop, value)); + } else if ( + value instanceof Array && + !this._isSimplySerializable(value[0]) + ) { + // array of objects + for (let i = 0; i < value.length; i++) { + out.push(""); + out.push(this._headerGroup(path + prop)); + out.push(...this._parse(value[i], `${path}${prop}.`)); + } + } else if (typeof value === "object") { + out.push(""); + out.push(this._header(path + prop)); + out.push(...this._parse(value, `${path}${prop}.`)); + } + } + out.push(""); + return out; + } + _isSimplySerializable(value: unknown): boolean { + return ( + typeof value === "string" || + typeof value === "number" || + value instanceof RegExp || + value instanceof Date || + value instanceof Array + ); + } + _header(title: string): string { + return `[${title}]`; + } + _headerGroup(title: string): string { + return `[[${title}]]`; + } + _declaration(title: string): string { + if (title.length > this.maxPad) { + this.maxPad = title.length; + } + return `${title} = `; + } + _arrayDeclaration(title: string, value: unknown[]): string { + return `${this._declaration(title)}${JSON.stringify(value)}`; + } + _strDeclaration(title: string, value: string): string { + return `${this._declaration(title)}"${value}"`; + } + _numberDeclaration(title: string, value: number): string { + switch (value) { + case Infinity: + return `${this._declaration(title)}inf`; + case -Infinity: + return `${this._declaration(title)}-inf`; + default: + return `${this._declaration(title)}${value}`; + } + } + _dateDeclaration(title: string, value: Date): string { + function dtPad(v: string, lPad: number = 2): string { + return pad(v, lPad, { char: "0" }); + } + let m = dtPad((value.getUTCMonth() + 1).toString()); + let d = dtPad(value.getUTCDate().toString()); + const h = dtPad(value.getUTCHours().toString()); + const min = dtPad(value.getUTCMinutes().toString()); + const s = dtPad(value.getUTCSeconds().toString()); + const ms = dtPad(value.getUTCMilliseconds().toString(), 3); + const fmtDate = `${value.getUTCFullYear()}-${m}-${d}T${h}:${min}:${s}.${ms}`; + return `${this._declaration(title)}${fmtDate}`; + } + _format(): string[] { + const rDeclaration = /(.*)\s=/; + const out = []; + for (let i = 0; i < this.output.length; i++) { + const l = this.output[i]; + // we keep empty entry for array of objects + if (l[0] === "[" && l[1] !== "[") { + // empty object + if (this.output[i + 1] === "") { + i += 1; + continue; + } + out.push(l); + } else { + const m = rDeclaration.exec(l); + if (m) { + out.push(l.replace(m[1], pad(m[1], this.maxPad, { side: "right" }))); + } else { + out.push(l); + } + } + } + // Cleaning multiple spaces + const cleanedOutput = []; + for (let i = 0; i < out.length; i++) { + const l = out[i]; + if (!(l === "" && out[i + 1] === "")) { + cleanedOutput.push(l); + } + } + return cleanedOutput; + } +} + +export function stringify(srcObj: object): string { + let out: string[] = []; + out = new Dumper(srcObj).dump(); + return out.join("\n"); +} + +export function parse(tomlString: string): object { + // File is potentially using EOL CRLF + tomlString = tomlString.replace(/\r\n/g, "\n").replace(/\\\n/g, "\n"); + return new Parser(tomlString).parse(); +} diff --git a/encoding/toml_test.ts b/encoding/toml_test.ts new file mode 100644 index 000000000..13d4f0b14 --- /dev/null +++ b/encoding/toml_test.ts @@ -0,0 +1,389 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +import { test } from "../testing/mod.ts"; +import { assertEquals } from "../testing/asserts.ts"; +import { existsSync } from "../fs/exists.ts"; +import { readFileStrSync } from "../fs/read_file_str.ts"; +import { parse, stringify } from "./toml.ts"; +import * as path from "../fs/path/mod.ts"; + +const testFilesDir = path.resolve("encoding", "testdata"); + +function parseFile(filePath: string): object { + if (!existsSync(filePath)) { + throw new Error(`File not found: ${filePath}`); + } + const strFile = readFileStrSync(filePath); + return parse(strFile); +} + +test({ + name: "[TOML] Strings", + fn(): void { + const expected = { + strings: { + str0: "deno", + str1: "Roses are not Deno\nViolets are not Deno either", + str2: "Roses are not Deno\nViolets are not Deno either", + str3: "Roses are not Deno\r\nViolets are not Deno either", + str4: 'this is a "quote"', + str5: "The quick brown\nfox jumps over\nthe lazy dog.", + str6: "The quick brown\nfox jumps over\nthe lazy dog.", + lines: + "The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved." + } + }; + const actual = parseFile(path.join(testFilesDir, "string.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] CRLF", + fn(): void { + const expected = { boolean: { bool1: true, bool2: false } }; + const actual = parseFile(path.join(testFilesDir, "CRLF.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Boolean", + fn(): void { + const expected = { boolean: { bool1: true, bool2: false } }; + const actual = parseFile(path.join(testFilesDir, "boolean.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Integer", + fn(): void { + const expected = { + integer: { + int1: 99, + int2: 42, + int3: 0, + int4: -17, + int5: 1000, + int6: 5349221, + int7: 12345, + hex1: "0xDEADBEEF", + hex2: "0xdeadbeef", + hex3: "0xdead_beef", + oct1: "0o01234567", + oct2: "0o755", + bin1: "0b11010110" + } + }; + const actual = parseFile(path.join(testFilesDir, "integer.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Float", + fn(): void { + const expected = { + float: { + flt1: 1.0, + flt2: 3.1415, + flt3: -0.01, + flt4: 5e22, + flt5: 1e6, + flt6: -2e-2, + flt7: 6.626e-34, + flt8: 224_617.445_991_228, + sf1: Infinity, + sf2: Infinity, + sf3: -Infinity, + sf4: NaN, + sf5: NaN, + sf6: NaN + } + }; + const actual = parseFile(path.join(testFilesDir, "float.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Arrays", + fn(): void { + const expected = { + arrays: { + data: [["gamma", "delta"], [1, 2]], + hosts: ["alpha", "omega"] + } + }; + const actual = parseFile(path.join(testFilesDir, "arrays.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Table", + fn(): void { + const expected = { + deeply: { + nested: { + object: { + in: { + the: { + toml: { + name: "Tom Preston-Werner" + } + } + } + } + } + }, + servers: { + alpha: { + ip: "10.0.0.1", + dc: "eqdc10" + }, + beta: { + ip: "10.0.0.2", + dc: "eqdc20" + } + } + }; + const actual = parseFile(path.join(testFilesDir, "table.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Simple", + fn(): void { + const expected = { + deno: "is", + not: "[node]", + regex: "", + NANI: "何?!", + comment: "Comment inside # the comment" + }; + const actual = parseFile(path.join(testFilesDir, "simple.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Datetime", + fn(): void { + const expected = { + datetime: { + odt1: new Date("1979-05-27T07:32:00Z"), + odt2: new Date("1979-05-27T00:32:00-07:00"), + odt3: new Date("1979-05-27T00:32:00.999999-07:00"), + odt4: new Date("1979-05-27 07:32:00Z"), + ld1: new Date("1979-05-27"), + lt1: "07:32:00", + lt2: "00:32:00.999999" + } + }; + const actual = parseFile(path.join(testFilesDir, "datetime.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Inline Table", + fn(): void { + const expected = { + inlinetable: { + nile: { + also: { + malevolant: { + creation: { + drum: { + kit: "Tama" + } + } + } + }, + derek: { + roddy: "drummer" + } + }, + name: { + first: "Tom", + last: "Preston-Werner" + }, + point: { + x: 1, + y: 2 + }, + dog: { + type: { + name: "pug" + } + }, + "tosin.abasi": "guitarist", + animal: { + as: { + leaders: "tosin" + } + } + } + }; + const actual = parseFile(path.join(testFilesDir, "inlineTable.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Array of Tables", + fn(): void { + const expected = { + bin: [ + { name: "deno", path: "cli/main.rs" }, + { name: "deno_core", path: "src/foo.rs" } + ], + nib: [{ name: "node", path: "not_found" }] + }; + const actual = parseFile(path.join(testFilesDir, "arrayTable.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Cargo", + fn(): void { + /* eslint-disable @typescript-eslint/camelcase */ + const expected = { + workspace: { members: ["./", "core"] }, + bin: [{ name: "deno", path: "cli/main.rs" }], + package: { name: "deno", version: "0.3.4", edition: "2018" }, + dependencies: { + deno_core: { path: "./core" }, + ansi_term: "0.11.0", + atty: "0.2.11", + dirs: "1.0.5", + flatbuffers: "0.5.0", + futures: "0.1.25", + getopts: "0.2.18", + http: "0.1.16", + hyper: "0.12.24", + "hyper-rustls": "0.16.0", + "integer-atomics": "1.0.2", + lazy_static: "1.3.0", + libc: "0.2.49", + log: "0.4.6", + rand: "0.6.5", + regex: "1.1.0", + remove_dir_all: "0.5.1", + ring: "0.14.6", + rustyline: "3.0.0", + serde_json: "1.0.38", + "source-map-mappings": "0.5.0", + tempfile: "3.0.7", + tokio: "0.1.15", + "tokio-executor": "0.1.6", + "tokio-fs": "0.1.5", + "tokio-io": "0.1.11", + "tokio-process": "0.2.3", + "tokio-threadpool": "0.1.11", + url: "1.7.2" + }, + target: { "cfg(windows)": { dependencies: { winapi: "0.3.6" } } } + }; + /* eslint-enable @typescript-eslint/camelcase */ + const actual = parseFile(path.join(testFilesDir, "cargo.toml")); + assertEquals(actual, expected); + } +}); + +test({ + name: "[TOML] Stringify", + fn(): void { + const src = { + foo: { bar: "deno" }, + this: { is: { nested: "denonono" } }, + arrayObjects: [{ stuff: "in" }, {}, { the: "array" }], + deno: "is", + not: "[node]", + regex: "", + NANI: "何?!", + comment: "Comment inside # the comment", + int1: 99, + int2: 42, + int3: 0, + int4: -17, + int5: 1000, + int6: 5349221, + int7: 12345, + flt1: 1.0, + flt2: 3.1415, + flt3: -0.01, + flt4: 5e22, + flt5: 1e6, + flt6: -2e-2, + flt7: 6.626e-34, + odt1: new Date("1979-05-01T07:32:00Z"), + odt2: new Date("1979-05-27T00:32:00-07:00"), + odt3: new Date("1979-05-27T00:32:00.999999-07:00"), + odt4: new Date("1979-05-27 07:32:00Z"), + ld1: new Date("1979-05-27"), + reg: /foo[bar]/, + sf1: Infinity, + sf2: Infinity, + sf3: -Infinity, + sf4: NaN, + sf5: NaN, + sf6: NaN, + data: [["gamma", "delta"], [1, 2]], + hosts: ["alpha", "omega"] + }; + const expected = `deno = "is" +not = "[node]" +regex = "" +NANI = "何?!" +comment = "Comment inside # the comment" +int1 = 99 +int2 = 42 +int3 = 0 +int4 = -17 +int5 = 1000 +int6 = 5349221 +int7 = 12345 +flt1 = 1 +flt2 = 3.1415 +flt3 = -0.01 +flt4 = 5e+22 +flt5 = 1000000 +flt6 = -0.02 +flt7 = 6.626e-34 +odt1 = 1979-05-01T07:32:00.000 +odt2 = 1979-05-27T07:32:00.000 +odt3 = 1979-05-27T07:32:00.999 +odt4 = 1979-05-27T07:32:00.000 +ld1 = 1979-05-27T00:00:00.000 +reg = "/foo[bar]/" +sf1 = inf +sf2 = inf +sf3 = -inf +sf4 = NaN +sf5 = NaN +sf6 = NaN +data = [["gamma","delta"],[1,2]] +hosts = ["alpha","omega"] + +[foo] +bar = "deno" + +[this.is] +nested = "denonono" + +[[arrayObjects]] +stuff = "in" + +[[arrayObjects]] + +[[arrayObjects]] +the = "array" +`; + const actual = stringify(src); + assertEquals(actual, expected); + } +}); -- cgit v1.2.3