[vcpkg] Add initial JSON support (#10521)

* [vcpkg] Add initial JSON support

This adds a JSON parser, as well as the amount of unicode support
required for JSON parsing to work according to the specification. In the
future, I hope to rewrite our existing XML files into JSON.

Additionally, as a drive-by, we've added the following:

* add /wd4800 to pragmas.h -- this is a "performance warning", for when
  you implicitly convert pointers or integers to bool, and shouldn't be
  an issue for us.
* Switched Parse::ParserBase to read unicode (as utf-8), as opposed to
  ASCII
* Building again under VCPKG_DEVELOPMENT_WARNINGS, yay!

2 files changed, 675 insertions, 0 deletions

diff --git a/toolsrc/src/vcpkg-test/json.cpp b/toolsrc/src/vcpkg-test/json.cpp
new file mode 100644
index 000000000..09f5d98fc
--- /dev/null
+++ b/toolsrc/src/vcpkg-test/json.cpp
@@ -0,0 +1,159 @@
+#include <catch2/catch.hpp>
+
+#include <iostream>
+#include <vcpkg/base/json.h>
+#include <vcpkg/base/unicode.h>
+
+// TODO: remove this once we switch to C++20 completely
+// This is the worst, but we also can't really deal with it any other way.
+#if __cpp_char8_t
+template<size_t Sz>
+static auto _u8_string_to_char_string(const char8_t (&literal)[Sz]) -> const char (&)[Sz]
+{
+    return reinterpret_cast<const char(&)[Sz]>(literal);
+}
+
+#define U8_STR(s) (::vcpkg::Unicode::_u8_string_to_char_string(u8"" s))
+#else
+#define U8_STR(s) (u8"" s)
+#endif
+
+namespace Json = vcpkg::Json;
+using Json::Value;
+
+static std::string mystringify(const Value& val) { return Json::stringify(val, Json::JsonStyle{}); }
+
+TEST_CASE ("JSON stringify weird strings", "[json]")
+{
+    vcpkg::StringView str = U8_STR("😀 😁 😂 🤣 😃 😄 😅 😆 😉");
+    REQUIRE(mystringify(Value::string(str)) == ('"' + str.to_string() + '"'));
+    REQUIRE(mystringify(Value::string("\xED\xA0\x80")) == "\"\\ud800\""); // unpaired surrogate
+}
+
+TEST_CASE ("JSON parse keywords", "[json]")
+{
+    auto res = Json::parse("true");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_boolean());
+    REQUIRE(res.get()->first.boolean());
+    res = Json::parse(" false ");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_boolean());
+    REQUIRE(!res.get()->first.boolean());
+    res = Json::parse(" null\t ");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_null());
+}
+
+TEST_CASE ("JSON parse strings", "[json]")
+{
+    auto res = Json::parse(R"("")");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string().size() == 0);
+
+    res = Json::parse(R"("\ud800")"); // unpaired surrogate
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == "\xED\xA0\x80");
+
+    const auto make_json_string = [] (vcpkg::StringView sv) {
+        return '"' + sv.to_string() + '"';
+    };
+    const vcpkg::StringView radical = U8_STR("⎷");
+    const vcpkg::StringView grin = U8_STR("😁");
+
+    res = Json::parse(R"("\uD83D\uDE01")"); // paired surrogates for grin
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == grin.to_string());
+
+    res = Json::parse(make_json_string(radical)); // character in BMP
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == radical);
+
+    res = Json::parse(make_json_string(grin)); // character above BMP
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_string());
+    REQUIRE(res.get()->first.string() == grin);
+}
+
+TEST_CASE ("JSON parse numbers", "[json]")
+{
+    auto res = Json::parse("0");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_number());
+    REQUIRE(res.get()->first.number() == 0);
+    res = Json::parse("12345");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_number());
+    REQUIRE(res.get()->first.number() == 12345);
+    res = Json::parse("-12345");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_number());
+    REQUIRE(res.get()->first.number() == -12345);
+    res = Json::parse("9223372036854775807"); // INT64_MAX
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_number());
+    REQUIRE(res.get()->first.number() == 9223372036854775807);
+    res = Json::parse("-9223372036854775808");
+    REQUIRE(res);
+    REQUIRE(res.get()->first.is_number());
+    REQUIRE(res.get()->first.number() == (-9223372036854775807 - 1)); // INT64_MIN (C++'s parser is fun)
+}
+
+TEST_CASE ("JSON parse arrays", "[json]")
+{
+    auto res = Json::parse("[]");
+    REQUIRE(res);
+    auto val = std::move(res.get()->first);
+    REQUIRE(val.is_array());
+    REQUIRE(val.array().size() == 0);
+
+    res = Json::parse("[123]");
+    REQUIRE(res);
+    val = std::move(res.get()->first);
+    REQUIRE(val.is_array());
+    REQUIRE(val.array().size() == 1);
+    REQUIRE(val.array()[0].is_number());
+    REQUIRE(val.array()[0].number() == 123);
+
+    res = Json::parse("[123, 456]");
+    REQUIRE(res);
+    val = std::move(res.get()->first);
+    REQUIRE(val.is_array());
+    REQUIRE(val.array().size() == 2);
+    REQUIRE(val.array()[0].is_number());
+    REQUIRE(val.array()[0].number() == 123);
+    REQUIRE(val.array()[1].is_number());
+    REQUIRE(val.array()[1].number() == 456);
+
+    res = Json::parse("[123, 456, [null]]");
+    REQUIRE(res);
+    val = std::move(res.get()->first);
+    REQUIRE(val.is_array());
+    REQUIRE(val.array().size() == 3);
+    REQUIRE(val.array()[2].is_array());
+    REQUIRE(val.array()[2].array().size() == 1);
+    REQUIRE(val.array()[2].array()[0].is_null());
+}
+
+TEST_CASE ("JSON parse objects", "[json]")
+{
+    auto res = Json::parse("{}");
+    REQUIRE(res);
+    auto val = std::move(res.get()->first);
+    REQUIRE(val.is_object());
+    REQUIRE(val.object().size() == 0);
+}
+
+TEST_CASE ("JSON parse full file", "[json]")
+{
+    vcpkg::StringView json =
+#include "large-json-document.json.inc"
+        ;
+
+    auto res = Json::parse(json);
+    REQUIRE(res);
+}
diff --git a/toolsrc/src/vcpkg-test/large-json-document.json.inc b/toolsrc/src/vcpkg-test/large-json-document.json.inc
new file mode 100644
index 000000000..b692d23ee
--- /dev/null
+++ b/toolsrc/src/vcpkg-test/large-json-document.json.inc
@@ -0,0 +1,516 @@
+// randomly generated by json-generator.com
+R"json([
+  {
+    "_id": "5e7a86c71cf688019f4c4b9f",
+    "index": 0,
+    "guid": "0e3c8a89-9960-4d87-8634-f9d3fcdaf735",
+    "isActive": false,
+    "balance": "$2,473.46",
+    "picture": "http://placehold.it/32x32",
+    "age": 21,
+    "eyeColor": "blue",
+    "name": {
+      "first": "Dana",
+      "last": "Romero"
+    },
+    "company": "NETPLODE",
+    "email": "dana.romero@netplode.info",
+    "phone": "+1 (981) 479-2769",
+    "address": "501 Cass Place, Strykersville, Missouri, 8329",
+    "about": "Incididunt in fugiat ipsum proident aliqua voluptate Lorem nostrud laboris ut velit minim. Dolor culpa magna tempor cupidatat. Id esse quis ipsum incididunt. Aliqua deserunt aliquip esse minim cillum esse aliquip veniam non labore dolor incididunt. Sint ea consectetur exercitation aliqua proident reprehenderit tempor ea eu. Amet ipsum labore magna elit.",
+    "registered": "Wednesday, September 4, 2019 8:29 AM",
+    "latitude": "-49.844379",
+    "longitude": "-5.565357",
+    "tags": [
+      "dolor",
+      "fugiat",
+      "ea",
+      "nisi",
+      "non"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Elena Suarez"
+      },
+      {
+        "id": 1,
+        "name": "Pruitt Leach"
+      },
+      {
+        "id": 2,
+        "name": "Pugh Robinson"
+      }
+    ],
+    "greeting": "Hello, Dana! You have 8 unread messages.",
+    "favoriteFruit": "apple"
+  },
+  {
+    "_id": "5e7a86c70efbf62ab5579408",
+    "index": 1,
+    "guid": "2c64c2d3-a830-4598-a399-f68f798ba6dc",
+    "isActive": true,
+    "balance": "$1,838.58",
+    "picture": "http://placehold.it/32x32",
+    "age": 33,
+    "eyeColor": "brown",
+    "name": {
+      "first": "Ladonna",
+      "last": "Willis"
+    },
+    "company": "JOVIOLD",
+    "email": "ladonna.willis@joviold.us",
+    "phone": "+1 (921) 591-2296",
+    "address": "441 Highland Place, Leyner, Pennsylvania, 1788",
+    "about": "Consequat deserunt nisi sit ex occaecat. Magna pariatur irure nisi duis laborum proident ipsum duis. Tempor qui consectetur consequat sunt proident ex ad id sint cupidatat sint.",
+    "registered": "Wednesday, January 13, 2016 6:03 PM",
+    "latitude": "-62.130182",
+    "longitude": "-102.884995",
+    "tags": [
+      "fugiat",
+      "ipsum",
+      "ut",
+      "pariatur",
+      "enim"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Gamble Rose"
+      },
+      {
+        "id": 1,
+        "name": "Olive Horn"
+      },
+      {
+        "id": 2,
+        "name": "Benita Ochoa"
+      }
+    ],
+    "greeting": "Hello, Ladonna! You have 6 unread messages.",
+    "favoriteFruit": "banana"
+  },
+  {
+    "_id": "5e7a86c71e30c95bbb0ff386",
+    "index": 2,
+    "guid": "04e45222-d785-461b-99be-b330585eb1a1",
+    "isActive": true,
+    "balance": "$3,591.60",
+    "picture": "http://placehold.it/32x32",
+    "age": 32,
+    "eyeColor": "brown",
+    "name": {
+      "first": "Lee",
+      "last": "Buckley"
+    },
+    "company": "TELEQUIET",
+    "email": "lee.buckley@telequiet.biz",
+    "phone": "+1 (897) 511-2132",
+    "address": "675 Argyle Road, Kempton, Ohio, 4411",
+    "about": "Sunt aliquip excepteur veniam fugiat consequat commodo ex est nulla laboris cillum enim. Laboris cupidatat et ipsum anim reprehenderit officia officia aute aliqua tempor. Incididunt sunt cupidatat mollit deserunt id nisi esse elit nisi est eiusmod aliquip. Lorem cillum ipsum quis aliquip laboris ex minim eu quis. Dolore incididunt officia labore enim Lorem in occaecat aliquip. Mollit ad duis non non qui et Lorem cillum.",
+    "registered": "Tuesday, January 16, 2018 3:14 PM",
+    "latitude": "-51.283144",
+    "longitude": "-112.569722",
+    "tags": [
+      "id",
+      "veniam",
+      "dolor",
+      "nulla",
+      "pariatur"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Dina Craft"
+      },
+      {
+        "id": 1,
+        "name": "Ashlee Ferrell"
+      },
+      {
+        "id": 2,
+        "name": "Mcbride Gill"
+      }
+    ],
+    "greeting": "Hello, Lee! You have 7 unread messages.",
+    "favoriteFruit": "banana"
+  },
+  {
+    "_id": "5e7a86c70e08bf0c278749f9",
+    "index": 3,
+    "guid": "0928ccac-e028-405a-a614-76628ba131b4",
+    "isActive": false,
+    "balance": "$2,867.88",
+    "picture": "http://placehold.it/32x32",
+    "age": 26,
+    "eyeColor": "green",
+    "name": {
+      "first": "Chen",
+      "last": "Rosa"
+    },
+    "company": "EXPOSA",
+    "email": "chen.rosa@exposa.me",
+    "phone": "+1 (956) 519-3064",
+    "address": "239 Amersfort Place, Fillmore, South Carolina, 2443",
+    "about": "Est ipsum cillum proident veniam voluptate enim sit eu excepteur veniam sit. Sunt aliqua qui incididunt id irure nulla qui. Et consequat ad anim proident minim dolor quis aliquip Lorem qui fugiat voluptate ex.",
+    "registered": "Wednesday, April 9, 2014 12:45 PM",
+    "latitude": "-15.222992",
+    "longitude": "76.730424",
+    "tags": [
+      "eiusmod",
+      "sit",
+      "do",
+      "aute",
+      "ea"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Huff Townsend"
+      },
+      {
+        "id": 1,
+        "name": "Stacie Downs"
+      },
+      {
+        "id": 2,
+        "name": "Liza Barron"
+      }
+    ],
+    "greeting": "Hello, Chen! You have 8 unread messages.",
+    "favoriteFruit": "banana"
+  },
+  {
+    "_id": "5e7a86c7086fc15efc387abc",
+    "index": 4,
+    "guid": "c71e62aa-8428-44cd-a2a5-7c30aaf963fe",
+    "isActive": true,
+    "balance": "$3,022.64",
+    "picture": "http://placehold.it/32x32",
+    "age": 32,
+    "eyeColor": "blue",
+    "name": {
+      "first": "Walton",
+      "last": "Mendez"
+    },
+    "company": "BUZZNESS",
+    "email": "walton.mendez@buzzness.tv",
+    "phone": "+1 (849) 560-2058",
+    "address": "507 Colonial Court, Collins, New York, 7941",
+    "about": "Et nisi in excepteur velit non incididunt sit. Consectetur magna sunt dolor eu Lorem adipisicing incididunt laborum consequat proident. Laboris ut dolor laboris esse ut dolor adipisicing ad fugiat commodo fugiat incididunt pariatur anim. Amet reprehenderit aute fugiat incididunt irure eu duis sint amet aliquip excepteur tempor dolore. Anim reprehenderit commodo irure sint et tempor occaecat fugiat ex commodo consectetur. Id ex dolor et culpa mollit. Voluptate magna est ex proident deserunt ullamco enim quis nulla cupidatat voluptate culpa exercitation Lorem.",
+    "registered": "Tuesday, May 9, 2017 2:38 PM",
+    "latitude": "86.083203",
+    "longitude": "57.386268",
+    "tags": [
+      "Lorem",
+      "aute",
+      "proident",
+      "eu",
+      "incididunt"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Byers Sims"
+      },
+      {
+        "id": 1,
+        "name": "Suzanne Gonzalez"
+      },
+      {
+        "id": 2,
+        "name": "Vicki Velasquez"
+      }
+    ],
+    "greeting": "Hello, Walton! You have 5 unread messages.",
+    "favoriteFruit": "banana"
+  },
+  {
+    "_id": "5e7a86c7dc9b82ffcb2868a2",
+    "index": 5,
+    "guid": "76f1c1cc-9164-43df-8858-633ee696df1c",
+    "isActive": true,
+    "balance": "$2,625.28",
+    "picture": "http://placehold.it/32x32",
+    "age": 40,
+    "eyeColor": "green",
+    "name": {
+      "first": "Wise",
+      "last": "Head"
+    },
+    "company": "FARMEX",
+    "email": "wise.head@farmex.com",
+    "phone": "+1 (850) 478-3280",
+    "address": "425 Kent Street, Witmer, New Jersey, 2411",
+    "about": "Velit amet fugiat enim occaecat do. Nulla sint officia anim ullamco. Ea quis excepteur excepteur enim ullamco. Amet aliqua mollit ad excepteur minim voluptate in velit sunt elit duis quis consequat nulla. Est dolor quis culpa aute id occaecat adipisicing mollit do consectetur fugiat. Mollit elit ex nostrud pariatur. Deserunt proident et voluptate occaecat labore occaecat Lorem exercitation est minim magna.",
+    "registered": "Monday, March 23, 2015 10:14 PM",
+    "latitude": "29.880281",
+    "longitude": "126.094567",
+    "tags": [
+      "enim",
+      "sunt",
+      "cupidatat",
+      "officia",
+      "aute"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Fuentes Tyler"
+      },
+      {
+        "id": 1,
+        "name": "Flora Massey"
+      },
+      {
+        "id": 2,
+        "name": "Manuela Parks"
+      }
+    ],
+    "greeting": "Hello, Wise! You have 8 unread messages.",
+    "favoriteFruit": "strawberry"
+  },
+  {
+    "_id": "5e7a86c7b3605b4ab198b25f",
+    "index": 6,
+    "guid": "818a0d46-9595-4066-a819-c93979220983",
+    "isActive": true,
+    "balance": "$3,193.77",
+    "picture": "http://placehold.it/32x32",
+    "age": 30,
+    "eyeColor": "brown",
+    "name": {
+      "first": "Baldwin",
+      "last": "Mcguire"
+    },
+    "company": "ZILCH",
+    "email": "baldwin.mcguire@zilch.io",
+    "phone": "+1 (803) 562-3968",
+    "address": "273 Homecrest Court, Caron, Virgin Islands, 7930",
+    "about": "Nostrud exercitation Lorem reprehenderit commodo aliquip. Exercitation exercitation proident aliquip et do cillum id ad ad reprehenderit ipsum elit nostrud. Occaecat velit sit commodo aliquip esse.",
+    "registered": "Saturday, January 14, 2017 3:13 AM",
+    "latitude": "-38.674801",
+    "longitude": "78.160951",
+    "tags": [
+      "reprehenderit",
+      "eu",
+      "magna",
+      "nulla",
+      "non"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Austin Gonzales"
+      },
+      {
+        "id": 1,
+        "name": "Polly Mcknight"
+      },
+      {
+        "id": 2,
+        "name": "Lucy Wagner"
+      }
+    ],
+    "greeting": "Hello, Baldwin! You have 7 unread messages.",
+    "favoriteFruit": "banana"
+  },
+  {
+    "_id": "5e7a86c700d93af1dcbe69f3",
+    "index": 7,
+    "guid": "09dc8fc1-207f-45f2-9d2c-f1e70e278e9a",
+    "isActive": true,
+    "balance": "$3,895.90",
+    "picture": "http://placehold.it/32x32",
+    "age": 21,
+    "eyeColor": "brown",
+    "name": {
+      "first": "Key",
+      "last": "Bolton"
+    },
+    "company": "MAGNINA",
+    "email": "key.bolton@magnina.net",
+    "phone": "+1 (918) 466-2785",
+    "address": "139 Blake Court, Chautauqua, Georgia, 3570",
+    "about": "Cupidatat excepteur reprehenderit eiusmod aute ea commodo ipsum pariatur dolore veniam adipisicing dolor. Excepteur ex in laborum cupidatat cillum cillum qui dolore consequat excepteur. Lorem deserunt eiusmod esse proident et ullamco reprehenderit ad ea. Cupidatat veniam deserunt magna eu labore ipsum et officia officia irure non eiusmod.",
+    "registered": "Wednesday, October 5, 2016 8:28 AM",
+    "latitude": "89.82294",
+    "longitude": "45.807834",
+    "tags": [
+      "et",
+      "excepteur",
+      "sunt",
+      "ea",
+      "irure"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Farmer Adkins"
+      },
+      {
+        "id": 1,
+        "name": "Summers Huffman"
+      },
+      {
+        "id": 2,
+        "name": "Lessie Holden"
+      }
+    ],
+    "greeting": "Hello, Key! You have 9 unread messages.",
+    "favoriteFruit": "strawberry"
+  },
+  {
+    "_id": "5e7a86c7a3fcbd25660a493d",
+    "index": 8,
+    "guid": "7d58a9c2-6940-499e-ba24-75227b1a09d9",
+    "isActive": true,
+    "balance": "$1,606.76",
+    "picture": "http://placehold.it/32x32",
+    "age": 31,
+    "eyeColor": "green",
+    "name": {
+      "first": "Gonzales",
+      "last": "Manning"
+    },
+    "company": "ORGANICA",
+    "email": "gonzales.manning@organica.name",
+    "phone": "+1 (993) 556-2745",
+    "address": "690 Regent Place, Rodman, Marshall Islands, 7114",
+    "about": "Magna ullamco voluptate nostrud et magna ea aute sint id quis proident ad excepteur ullamco. Aliquip nostrud qui quis duis occaecat commodo laborum labore aute. Mollit ullamco in qui eu voluptate dolore aute mollit sint do sit nulla aliqua. Occaecat laborum ex velit ea ex ad eiusmod enim fugiat.",
+    "registered": "Monday, December 1, 2014 6:32 AM",
+    "latitude": "48.780262",
+    "longitude": "-75.333042",
+    "tags": [
+      "non",
+      "laborum",
+      "et",
+      "Lorem",
+      "id"
+    ],
+    "range": [
+      0,
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9
+    ],
+    "friends": [
+      {
+        "id": 0,
+        "name": "Hester Finch"
+      },
+      {
+        "id": 1,
+        "name": "Tia Cooley"
+      },
+      {
+        "id": 2,
+        "name": "Cathryn Howe"
+      }
+    ],
+    "greeting": "Hello, Gonzales! You have 5 unread messages.",
+    "favoriteFruit": "apple"
+  }
+])json"