From 165907550c8f6ce7506beef591f55cd3f8458d78 Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Fri, 2 Aug 2019 09:52:39 -0700 Subject: Update tests, and add documentation! (#7506) This PR does the following: * fix tests -- now, they're always built in the CMake scripts, and they work on VS2015 *add a new flag, BUILD_TESTING, which allows one to turn off testing builds * Add documentation for running tests --- docs/tool-maintainers/testing.md | 162 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 docs/tool-maintainers/testing.md (limited to 'docs/tool-maintainers') diff --git a/docs/tool-maintainers/testing.md b/docs/tool-maintainers/testing.md new file mode 100644 index 000000000..28cc9e099 --- /dev/null +++ b/docs/tool-maintainers/testing.md @@ -0,0 +1,162 @@ +Testing +======= + +Testing vcpkg is important whenever one makes changes to the tool itself, and +writing new tests and keeping them up to date is also very important. If one's +code is subtly broken, we'd rather find it out right away than a few weeks down +the line when someone complains! + +Running Tests +------------- + +Before anything else, we should know whether you can actually run the tests! +All you should need is a way to build vcpkg -- anything will do! All you have to +do is follow the guide 😄 + +With `$VCPKG_DIRECTORY` being the directory where you have cloned vcpkg, create +a build directory in `$VCPKG_DIRECTORY/toolsrc` (commonly named `out`), and +`cd` into it. Make sure to clean it out if it already exists! + +```sh +$ cmake .. -DCMAKE_BUILD_TYPE=Debug -G Ninja +$ cmake --build . +$ ./vcpkg-test # ./vcpkg-test [$SPECIFIC_TEST] for a specific set of tests +$ # i.e., ./vcpkg-test [arguments] +``` + +If you make any modifications to `vcpkg`, you'll have to do the +`cmake --build .` step again. + +Writing Tests +------------- + +In your journey to write new tests, and to modify existing tests, reading the +[Catch2 documentation] will be very helpful! Come back after reading those 😀 + +You'll want to place your tests in one of the existing files, or, if it doesn't +belong in any of those, in a [new file](#adding-new-test-files). + +The layout of these tests is as follows: + +```cpp +// ... includes + +TEST_CASE("Name of test", "[filename without the .cpp]") { + // setup and the like + REQUIRE(some boolean expression); +} + +// etc. +``` + +You want to give these test cases good, descriptive, unique names, like +`SourceParagraph construct minimum` -- it doesn't need to be extremely clear +english, and shorthand is good, but make sure it's clear what the test is from +the name. For the latter parameter, known as "tags", you should at least put the +name of the file which the test case is in -- e.g., in `arguments.cpp`, you'd +tag all of the test cases with `[arguments]`. + +If you wish to add helper functions, make sure to place them in an anonymous +namespace -- this will ensure that they don't trample over anybody else's +space. Additionally, there are a few helper functions that live in +`` and `src/vcpkg-test/util.cpp` -- make sure to look into +them so that you're not rewriting functionality. + +That should be all you need to know to start writing your own tests! +Remember to check out the [Catch2 documentation] +if you'd like to get more advanced with your tests, +and good luck on your testing journey! + +Adding New Test Files +--------------------- + +Adding new test files should be easy and straightforward. All it requires is +creating a new source file in `toolsrc/src/vcpkg-test`, and then rerunning +`CMake` in order to pick up the glob changes. + +### Example + +Let's try writing a new test file called `example` (very creative, I know). + +First, we should create a file, `example.cpp`, in `toolsrc/src/vcpkg-test`: + +```cpp +// vcpkg-test/example.cpp +#include +``` + +This is the minimum file needed for tests; let's rebuild our CMake directory. +You'll have to clean out the existing `out` directory for CMake to rerun +globbing. + +```sh +$ cmake .. -DCMAKE_BUILD_TYPE=Debug -G Ninja +# ... +-- Build files have been written to: $VCPKG_DIRECTORY/toolsrc/out +$ cmake --build . +[80/80] Linking CXX executable vcpkg.exe +``` + +Okay, now let's make sure this worked; add a test case to `example.cpp`: + +```cpp +TEST_CASE("Example 1 - fail", "[example]") { + REQUIRE(false); +} +``` + +Now build the tests again, and run them: + +```sh +$ cmake --build . +[2/2] Linking CXX executable vcpkg-test.exe +$ ./vcpkg-test + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +vcpkg-test.exe is a Catch v2.9.1 host application. +Run with -? for options + +------------------------------------------------------------------------------- +Example 1 - fail +------------------------------------------------------------------------------- +$VCPKG_DIRECTORY/toolsrc/src/vcpkg-test/example.cpp(3) +............................................................................... + +$VCPKG_DIRECTORY/toolsrc/src/vcpkg-test/example.cpp(14): FAILED: + REQUIRE( false ) + +=============================================================================== +test cases: 102 | 101 passed | 1 failed +assertions: 3611 | 3610 passed | 1 failed +``` + +Hopefully, that worked! It should compile correctly, and have one failing test. +Now let's try a more complex test, after deleting the old one; + +```cpp +// add #include to the top of the file +namespace Strings = vcpkg::Strings; + +TEST_CASE("Example 2 - success", "[example]") { + std::string hello = "Hello"; + REQUIRE(Strings::case_insensitive_ascii_equals(hello, "hELLo")); + REQUIRE_FALSE(Strings::case_insensitive_ascii_starts_with(hello, "E")); +} +``` + +Now compile and build the tests, and this time let's only run our example tests: + +```sh +$ cmake --build . +[2/2] Linking CXX executable vcpkg-test.exe +$ ./vcpkg-test [example] +Filters: [example] +=============================================================================== +All tests passed (2 assertions in 1 test case) +``` + +Hopefully you have one test running and succeeding! If you have that, you have +succeeded at adding a new file to vcpkg's tests. Congratulations! Have fun on +the rest of your journey 🐱‍👤😁 + +[Catch2 documentation]: https://github.com/catchorg/Catch2/blob/master/docs/tutorial.md#top -- cgit v1.2.3 From 2a81a2d3220353fa8a5890e234b77cadc2c4b589 Mon Sep 17 00:00:00 2001 From: nicole mazzuca Date: Sat, 24 Aug 2019 11:36:12 -0700 Subject: [vcpkg docs] More tool maintainer docs! (#7821) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [vcpkg docs] Add benchmarking 🏎 to the docs! Also, minor changes to the testing docs. * add documentation for the layout of the project --- docs/tool-maintainers/benchmarking.md | 195 ++++++++++++++++++++++++++++++++++ docs/tool-maintainers/layout.md | 85 +++++++++++++++ docs/tool-maintainers/testing.md | 36 +++---- 3 files changed, 293 insertions(+), 23 deletions(-) create mode 100644 docs/tool-maintainers/benchmarking.md create mode 100644 docs/tool-maintainers/layout.md (limited to 'docs/tool-maintainers') diff --git a/docs/tool-maintainers/benchmarking.md b/docs/tool-maintainers/benchmarking.md new file mode 100644 index 000000000..e0295be50 --- /dev/null +++ b/docs/tool-maintainers/benchmarking.md @@ -0,0 +1,195 @@ +# Benchmarking + +Benchmarking new code against old code is extremely important whenever making +large changes to how something works. If you are attempting to make something +faster, and you end up slowing it down, you'll never know if you don't +benchmark! We have benchmarks in the `toolsrc/src/vcpkg-test` directory, just +like the tests -- they're treated as a special kind of test. + +## Running Benchmarks + +Unlike normal tests, benchmarks are hidden behind a special define -- `CATCH_CONFIG_ENABLE_BENCHMARKING` -- so that you never try to run benchmarks +unless you specifically want to. This is because benchmarks actually take quite +a long time! However, if you want to run benchmarks (and I recommend running +only specific benchmarks at a time), you can do so by passing the +`VCPKG_ENABLE_BENCHMARKING` option at cmake configure time. + +```sh +$ cmake -B toolsrc/out -S toolsrc -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DVCPKG_BUILD_BENCHMARKING=On + +-- The C compiler identification is MSVC 19.22.27905.0 +-- The CXX compiler identification is MSVC 19.22.27905.0 +-- Check for working C compiler: C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.22.27905/bin/Hostx64/x64/cl.exe +-- Check for working C compiler: C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.22.27905/bin/Hostx64/x64/cl.exe -- works +-- Detecting C compiler ABI info +-- Detecting C compiler ABI info - done +-- Detecting C compile features +-- Detecting C compile features - done +-- Check for working CXX compiler: C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.22.27905/bin/Hostx64/x64/cl.exe +-- Check for working CXX compiler: C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC/14.22.27905/bin/Hostx64/x64/cl.exe -- works +-- Detecting CXX compiler ABI info +-- Detecting CXX compiler ABI info - done +-- Detecting CXX compile features +-- Detecting CXX compile features - done +-- Looking for pthread.h +-- Looking for pthread.h - not found +-- Found Threads: TRUE +-- Configuring done +-- Generating done +-- Build files have been written to: C:/Users/t-nimaz/src/vcpkg/toolsrc/out + +$ cmake --build toolsrc/out + +[0/2] Re-checking globbed directories... +[80/80] Linking CXX executable vcpkg-test.exe +``` + +You can then run benchmarks easily with the following command (which run the +files benchmarks): + +```sh +$ ./toolsrc/out/vcpkg-test [!benchmark][file] +``` + +You can switch out `[file]` for a different set -- `[hash]`, for example. + +## Writing Benchmarks + +First, before anything else, I recommend reading the +[benchmarking documentation] at Catch2's repository. + +Now, after that, let's say that you wanted to benchmark, say, our ASCII +case-insensitive string compare against your new implementation. We place +benchmarks for code in the same file as their tests, so open +`vcpkg-test/strings.cpp`, and add the following at the bottom: + +```cpp +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) +TEST_CASE ("case insensitive ascii equals: benchmark", "[strings][!benchmark]") +{ + BENCHMARK("qwertyuiop") { + return vcpkg::Strings::case_insensitive_ascii_equals("qwertyuiop", "QWERTYUIOP"); + }; +} +#endif +``` + +Remember the `;` at the end of the benchmark -- it's not required for +`TEST_CASE`s, but is for `BENCHMARK`s. + +Now, let's rebuild and run: + +```sh +$ cmake --build toolsrc/out +[0/2] Re-checking globbed directories... +[2/2] Linking CXX executable vcpkg-test.exe +$ ./toolsrc/out/vcpkg-test [strings][!benchmark] +Filters: [strings][!benchmark] + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +vcpkg-test.exe is a Catch v2.9.1 host application. +Run with -? for options + +------------------------------------------------------------------------------- +case insensitive ascii equals: benchmark +------------------------------------------------------------------------------- +C:\Users\t-nimaz\src\vcpkg\toolsrc\src\vcpkg-test\strings.cpp(36) +............................................................................... + +benchmark name samples iterations estimated + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +qwertyuiop 100 2088 3.9672 ms + 25 ns 24 ns 26 ns + 6 ns 5 ns 8 ns + + +=============================================================================== +test cases: 1 | 1 passed +assertions: - none - +``` + +You've now written your first benchmark! + +But wait. This seems kind of silly. Benchmarking the comparison of literal +strings is great and all, but could we make it a little more realistic? + +This is where `BENCHMARK_ADVANCED` comes in. `BENCHMARK_ADVANCED` allows one to +write a benchmark that has a little setup to it without screwing up the numbers. +Let's try it now: + +```cpp +TEST_CASE ("case insensitive ascii equals: benchmark", "[strings][!benchmark]") +{ + BENCHMARK_ADVANCED("equal strings")(Catch::Benchmark::Chronometer meter) + { + std::vector strings; + strings.resize(meter.runs()); + std::mt19937_64 urbg; + std::uniform_int_distribution data_generator; + + std::generate(strings.begin(), strings.end(), [&] { + std::string result; + for (std::size_t i = 0; i < 1000; ++i) + { + result += vcpkg::Strings::b32_encode(data_generator(urbg)); + } + + return result; + }); + + meter.measure( + [&](int run) { return vcpkg::Strings::case_insensitive_ascii_equals(strings[run], strings[run]); }); + }; +} +``` + +Then, run it again! + +```sh +$ cmake --build toolsrc/out +[0/2] Re-checking globbed directories... +[2/2] Linking CXX executable vcpkg-test.exe +$ toolsrc/out/vcpkg-test [strings][!benchmark] +Filters: [strings][!benchmark] + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +vcpkg-test.exe is a Catch v2.9.1 host application. +Run with -? for options + +------------------------------------------------------------------------------- +case insensitive ascii equals: benchmark +------------------------------------------------------------------------------- +C:\Users\t-nimaz\src\vcpkg\toolsrc\src\vcpkg-test\strings.cpp(36) +............................................................................... + +benchmark name samples iterations estimated + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +equal strings 100 2 5.4806 ms + 22.098 us 21.569 us 23.295 us + 3.842 us 2.115 us 7.41 us + + +=============================================================================== +test cases: 1 | 1 passed +assertions: - none - +``` + +And now you have a working benchmark to test the speed of the existing code, and +of new code! + +If you're writing a lot of benchmarks that follow the same sort of pattern, with +some differences in constants, look into `vcpkg-test/files.cpp`'s benchmarks -- +there are a lot of things one can do to make writing new benchmarks really easy. + +If you wish to add a benchmark for a piece of code that has not yet been tested, +please read the [testing documentation], and please write some unit tests. +The speed of your code isn't very important if it doesn't work at all! + +[benchmarking documentation]: https://github.com/catchorg/Catch2/blob/master/docs/benchmarks.md#top +[testing documentation]: ./testing.md#adding-new-test-files diff --git a/docs/tool-maintainers/layout.md b/docs/tool-maintainers/layout.md new file mode 100644 index 000000000..9779b0434 --- /dev/null +++ b/docs/tool-maintainers/layout.md @@ -0,0 +1,85 @@ +# Layout of the vcpkg source tree + +All vcpkg sources and build systems are in `toolsrc`. If you'd like to +contribute to the vcpkg tool itself, most of your time will be spent in here. + +## Build Files + +These are the files used to build and configure the project. In order to build +with CMake, the only files you should be interested in are `CMakeLists.txt`, and +`.clang-format`; in order to build with msbuild or the Visual Studio IDE, you +will be interested in `dirs.proj` or `vcpkg.sln`. However, if you add or remove +files, you will need to edit the MSBuild project files in the `vcpkg*` +directories no matter what system you use. + +### Top Level + +We have six files in this directory -- one `.clang-format` file, one +`CMakeLists.txt` file, three Visual Studio files, and `VERSION.txt`. + + - `.clang-format`: This is where we store the formatting settings of the + project. If you want to format the project, you can use the `format` target + with the CMake build system. + - `CMakeLists.txt`: This is where the CMake build system definition lives. If + you want to modify how one builds the project, or add a target, you can do + it here. + - `VERSION.txt`: This is a file which tells `vcpkg` to tell the user to + rebuild. If this version is different from the version when the user built + the binary (for example, after a `git pull` or a `vcpkg update`), then + `vcpkg` will print a message to re-bootstrap. This is updated whenever major + changes are made to the `vcpkg` tool. + - The Visual Studio files: + - `vcpkg.natvis`: NATVIS files allow one to visualize objects of user + defined type in the debugger -- this one contains the definitions for + `vcpkg`'s types. + - `dirs.proj`: This is how one builds with `msbuild` without calling into + the IDE. + - `vcpkg.sln`: The solution file is how one opens the project in the VS IDE. + +### `vcpkg`, `vcpkglib`, `vcpkgmetricsuploader`, and `vcpkgtest` + +These four contain exactly one `.vcxproj` and one +`.vcxproj.filters`. The `.vcxproj` file contains the source files +and the `.vcxproj.filters` contains information on how Visual Studio +should lay out the project's source files in the IDE's project view. + +`vcpkgtest` should not be touched. It's likely that it will be deleted soon. If +you want to test your code, use the cmake build system. + +## Source Files + +If you're modifying the project, it's likely that these are the directories that +you're going to deal with. + +### `include` + +There's one file in here -- `pch.h`. This contains most of the C++ standard +library, and acts as a [precompiled header]. You can read more at the link. + +There are three directories: + + - `catch2` -- This contains the single-header library [catch2]. We use this + library for both [testing] and [benchmarking]. + - `vcpkg` -- This contains the header files for the `vcpkg` project. All of + the interfaces for building, installing, and generally "port stuff" live + here. + - `vcpkg/base` -- This contains the interfaces for the + "vcpkg standard library" -- file handling, hashing, strings, + `Span`, printing, etc. + - `vcpkg-test` -- This contains the interfaces for any common utilities + required by the tests. + +### `src` + +The source files live here. `pch.cpp` is the source file for the +[precompiled header]; `vcpkg.cpp` is where the `vcpkg` binary lives; and +`vcpkgmetricsuploader.cpp` is where the metrics uploader lives. + +The interesting files live in the `vcpkg` and `vcpkg-test` directories. In +`vcpkg`, you have the implementation for the interfaces that live in +`include/vcpkg`; and in `vcpkg-test`, you have the tests and benchmarks. + +[precompiled header]: https://en.wikipedia.org/wiki/Precompiled_header +[catch2]: https://github.com/catchorg/Catch2 +[testing]: ./testing.md +[benchmarking]: ./benchmarking.md diff --git a/docs/tool-maintainers/testing.md b/docs/tool-maintainers/testing.md index 28cc9e099..0284a2650 100644 --- a/docs/tool-maintainers/testing.md +++ b/docs/tool-maintainers/testing.md @@ -1,13 +1,11 @@ -Testing -======= +# Testing Testing vcpkg is important whenever one makes changes to the tool itself, and writing new tests and keeping them up to date is also very important. If one's code is subtly broken, we'd rather find it out right away than a few weeks down the line when someone complains! -Running Tests -------------- +## Running Tests Before anything else, we should know whether you can actually run the tests! All you should need is a way to build vcpkg -- anything will do! All you have to @@ -27,8 +25,7 @@ $ # i.e., ./vcpkg-test [arguments] If you make any modifications to `vcpkg`, you'll have to do the `cmake --build .` step again. -Writing Tests -------------- +## Writing Tests In your journey to write new tests, and to modify existing tests, reading the [Catch2 documentation] will be very helpful! Come back after reading those 😀 @@ -42,8 +39,8 @@ The layout of these tests is as follows: // ... includes TEST_CASE("Name of test", "[filename without the .cpp]") { - // setup and the like - REQUIRE(some boolean expression); + // setup and the like + REQUIRE(some boolean expression); } // etc. @@ -67,12 +64,10 @@ Remember to check out the [Catch2 documentation] if you'd like to get more advanced with your tests, and good luck on your testing journey! -Adding New Test Files ---------------------- +## Adding New Test Files Adding new test files should be easy and straightforward. All it requires is -creating a new source file in `toolsrc/src/vcpkg-test`, and then rerunning -`CMake` in order to pick up the glob changes. +creating a new source file in `toolsrc/src/vcpkg-test`. ### Example @@ -85,14 +80,9 @@ First, we should create a file, `example.cpp`, in `toolsrc/src/vcpkg-test`: #include ``` -This is the minimum file needed for tests; let's rebuild our CMake directory. -You'll have to clean out the existing `out` directory for CMake to rerun -globbing. +This is the minimum file needed for tests; let's rebuild! ```sh -$ cmake .. -DCMAKE_BUILD_TYPE=Debug -G Ninja -# ... --- Build files have been written to: $VCPKG_DIRECTORY/toolsrc/out $ cmake --build . [80/80] Linking CXX executable vcpkg.exe ``` @@ -101,7 +91,7 @@ Okay, now let's make sure this worked; add a test case to `example.cpp`: ```cpp TEST_CASE("Example 1 - fail", "[example]") { - REQUIRE(false); + REQUIRE(false); } ``` @@ -123,7 +113,7 @@ $VCPKG_DIRECTORY/toolsrc/src/vcpkg-test/example.cpp(3) ............................................................................... $VCPKG_DIRECTORY/toolsrc/src/vcpkg-test/example.cpp(14): FAILED: - REQUIRE( false ) + REQUIRE( false ) =============================================================================== test cases: 102 | 101 passed | 1 failed @@ -138,9 +128,9 @@ Now let's try a more complex test, after deleting the old one; namespace Strings = vcpkg::Strings; TEST_CASE("Example 2 - success", "[example]") { - std::string hello = "Hello"; - REQUIRE(Strings::case_insensitive_ascii_equals(hello, "hELLo")); - REQUIRE_FALSE(Strings::case_insensitive_ascii_starts_with(hello, "E")); + std::string hello = "Hello"; + REQUIRE(Strings::case_insensitive_ascii_equals(hello, "hELLo")); + REQUIRE_FALSE(Strings::case_insensitive_ascii_starts_with(hello, "E")); } ``` -- cgit v1.2.3