Add testing of network functionality

6 files changed, 524 insertions, 9 deletions

diff --git a/data/Makefile.am b/data/Makefile.am
index 2a8eb285..59ab4e83 100644
--- a/data/Makefile.am
+++ b/data/Makefile.am
@@ -80,6 +80,7 @@ EXTRA_DIST = GL27 nad.lst nad27 nad83 \
 		tests/test_hgrid_with_subgrid.tif \
 		tests/test_hgrid_with_subgrid_no_grid_name.tif \
 		tests/subset_of_gr3df97a.tif \
+		tests/egm96_15_uncompressed_truncated.tif \
 		null \
 		generate_all_sql_in.cmake sql_filelist.cmake \
 		$(SQL_ORDERED_LIST)
diff --git a/data/tests/egm96_15_uncompressed_truncated.tif b/data/tests/egm96_15_uncompressed_truncated.tif
new file mode 100644
index 00000000..bd34a7e2
--- /dev/null
+++ b/data/tests/egm96_15_uncompressed_truncated.tif
diff --git a/src/filemanager.cpp b/src/filemanager.cpp
index 2dfcd6b6..cc692616 100644
--- a/src/filemanager.cpp
+++ b/src/filemanager.cpp
@@ -260,10 +260,7 @@ NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) {
 
 // ---------------------------------------------------------------------------
 
-void NetworkChunkCache::clear()
-{
-    cache_.clear();
-}
+void NetworkChunkCache::clear() { cache_.clear(); }
 
 // ---------------------------------------------------------------------------
 
@@ -672,10 +669,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) {
 
 // ---------------------------------------------------------------------------
 
-void FileManager::clearCache()
-{
-    gNetworkChunkCache.clear();
-}
+void FileManager::clearCache() { gNetworkChunkCache.clear(); }
 
 // ---------------------------------------------------------------------------
 
diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt
index 841d72b3..23d54663 100644
--- a/test/unit/CMakeLists.txt
+++ b/test/unit/CMakeLists.txt
@@ -147,3 +147,18 @@ target_link_libraries(gie_self_tests
 add_test(NAME gie_self_tests COMMAND gie_self_tests)
 set_property(TEST gie_self_tests
         PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data")
+
+
+add_executable(test_network
+  main.cpp
+  test_network.cpp)
+if(CURL_FOUND)
+  include_directories(${CURL_INCLUDE_DIR})
+  target_link_libraries(test_network ${CURL_LIBRARY})
+endif()
+target_link_libraries(test_network
+  GTest::gtest
+  ${PROJ_LIBRARIES})
+add_test(NAME test_network COMMAND test_network)
+set_property(TEST test_network
+        PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data")
diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am
index 57a03ca8..23ff5076 100644
--- a/test/unit/Makefile.am
+++ b/test/unit/Makefile.am
@@ -17,6 +17,7 @@ noinst_PROGRAMS += proj_context_test
 noinst_PROGRAMS += test_cpp_api
 noinst_PROGRAMS += gie_self_tests
 noinst_PROGRAMS += include_proj_h_from_c
+noinst_PROGRAMS += test_network
 
 pj_transform_test_SOURCES = pj_transform_test.cpp main.cpp
 pj_transform_test_LDADD = ../../src/libproj.la @GTEST_LIBS@
@@ -62,4 +63,11 @@ gie_self_tests-check: gie_self_tests
 
 include_proj_h_from_c_SOURCES = include_proj_h_from_c.c
 
-check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check
+test_network_SOURCES = test_network.cpp main.cpp
+test_network_CXXFLAGS = @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@
+test_network_LDADD = ../../src/libproj.la @GTEST_LIBS@ @CURL_LIBS@
+
+test_network-check: test_network
+	PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network
+
+check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check test_network-check
diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp
new file mode 100644
index 00000000..6879e40d
--- /dev/null
+++ b/test/unit/test_network.cpp
@@ -0,0 +1,497 @@
+/******************************************************************************
+ *
+ * Project:  PROJ
+ * Purpose:  Test networking
+ * Author:   Even Rouault <even dot rouault at spatialys dot com>
+ *
+ ******************************************************************************
+ * Copyright (c) 2019, Even Rouault <even dot rouault at spatialys dot com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ ****************************************************************************/
+
+#include "gtest_include.h"
+
+#include <memory>
+#include <stdio.h>
+
+#include "proj_internal.h"
+#include <proj.h>
+
+#ifdef CURL_ENABLED
+#include <curl/curl.h>
+#endif
+
+namespace {
+
+// ---------------------------------------------------------------------------
+
+#ifdef CURL_ENABLED
+
+static bool networkAccessOK = false;
+
+static size_t noop_curl_write_func(void *, size_t, size_t nmemb, void *) {
+    return nmemb;
+}
+
+TEST(networking, initial_check) {
+    CURL *hCurlHandle = curl_easy_init();
+    if (!hCurlHandle)
+        return;
+    curl_easy_setopt(hCurlHandle, CURLOPT_URL,
+                     "https://cdn.proj.org/ntf_r93.tif");
+
+    curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, "0-1");
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, noop_curl_write_func);
+
+    curl_easy_perform(hCurlHandle);
+
+    long response_code = 0;
+    curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
+
+    curl_easy_cleanup(hCurlHandle);
+
+    networkAccessOK = (response_code == 206);
+    if (!networkAccessOK) {
+        fprintf(stderr, "network access not working");
+    }
+}
+
+#endif
+
+// ---------------------------------------------------------------------------
+
+TEST(networking, basic) {
+    auto P = proj_create(
+        PJ_DEFAULT_CTX,
+        "+proj=pipeline "
+        "+step +proj=unitconvert +xy_in=deg +xy_out=rad "
+        "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif "
+        "+step +proj=unitconvert +xy_in=rad +xy_out=deg ");
+#ifdef CURL_ENABLED
+    if (networkAccessOK) {
+        ASSERT_NE(P, nullptr);
+    } else {
+        ASSERT_EQ(P, nullptr);
+        return;
+    }
+    double lon = 2;
+    double lat = 49;
+    proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double),
+                       1, nullptr, 0, 0, nullptr, 0, 0);
+    EXPECT_NEAR(lon, 1.9992776848, 1e-10);
+    EXPECT_NEAR(lat, 48.9999322600, 1e-10);
+
+    proj_destroy(P);
+#else
+    ASSERT_EQ(P, nullptr);
+#endif
+}
+
+// ---------------------------------------------------------------------------
+
+#ifdef CURL_ENABLED
+
+static void silent_logger(void *, int, const char *) {}
+
+TEST(networking, curl_invalid_resource) {
+    auto ctx = proj_context_create();
+    proj_log_func(ctx, nullptr, silent_logger);
+    auto P = proj_create(
+        ctx, "+proj=hgridshift +grids=https://i_do_not.exist/my.tif");
+    proj_context_destroy(ctx);
+    ASSERT_EQ(P, nullptr);
+}
+#endif
+
+// ---------------------------------------------------------------------------
+
+struct Event {
+    virtual ~Event();
+    std::string type{};
+    PJ_CONTEXT *ctx = nullptr;
+};
+
+Event::~Event() = default;
+
+struct OpenEvent : public Event {
+    OpenEvent() { type = "OpenEvent"; }
+
+    std::string url{};
+    unsigned long long offset = 0;
+    size_t size_to_read = 0;
+    std::vector<unsigned char> response{};
+    int file_id = 0;
+};
+
+struct CloseEvent : public Event {
+    CloseEvent() { type = "CloseEvent"; }
+
+    int file_id = 0;
+};
+
+struct GetHeaderValueEvent : public Event {
+    GetHeaderValueEvent() { type = "GetHeaderValueEvent"; }
+};
+
+struct GetFileSizeEvent : public Event {
+    GetFileSizeEvent() { type = "GetFileSizeEvent"; }
+};
+
+struct ReadRangeEvent : public Event {
+    ReadRangeEvent() { type = "ReadRangeEvent"; }
+
+    unsigned long long offset = 0;
+    size_t size_to_read = 0;
+    std::vector<unsigned char> response{};
+    int file_id = 0;
+};
+
+struct GetLastErrorEvent : public Event {
+    GetLastErrorEvent() { type = "GetLastErrorEvent"; }
+};
+
+struct File {};
+
+struct ExchangeWithCallback {
+    std::vector<std::unique_ptr<Event>> events{};
+    size_t nextEvent = 0;
+    bool error = false;
+    std::map<int, PROJ_NETWORK_HANDLE *> mapIdToHandle{};
+
+    bool allConsumedAndNoError() const {
+        return nextEvent == events.size() && !error;
+    }
+};
+
+static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url,
+                                     unsigned long long offset,
+                                     size_t size_to_read, void *buffer,
+                                     size_t *out_size_read, void *user_data) {
+    auto exchange = static_cast<ExchangeWithCallback *>(user_data);
+    if (exchange->error)
+        return nullptr;
+    if (exchange->nextEvent >= exchange->events.size()) {
+        fprintf(stderr, "unexpected call to open(%s, %ld, %ld)\n", url,
+                (long)offset, (long)size_to_read);
+        exchange->error = true;
+        return nullptr;
+    }
+    auto openEvent =
+        dynamic_cast<OpenEvent *>(exchange->events[exchange->nextEvent].get());
+    if (!openEvent) {
+        fprintf(stderr, "unexpected call to open(%s, %ld, %ld). "
+                        "Was expecting a %s event\n",
+                url, (long)offset, (long)size_to_read,
+                exchange->events[exchange->nextEvent]->type.c_str());
+        exchange->error = true;
+        return nullptr;
+    }
+    exchange->nextEvent++;
+    if (openEvent->ctx != ctx || openEvent->url != url ||
+        openEvent->offset != offset ||
+        openEvent->size_to_read != size_to_read) {
+        fprintf(stderr, "wrong call to open(%s, %ld, %ld). Was expecting "
+                        "open(%s, %ld, %ld)\n",
+                url, (long)offset, (long)size_to_read, openEvent->url.c_str(),
+                (long)openEvent->offset, (long)openEvent->size_to_read);
+        exchange->error = true;
+        return nullptr;
+    }
+    memcpy(buffer, openEvent->response.data(), openEvent->response.size());
+    *out_size_read = openEvent->response.size();
+
+    auto handle = reinterpret_cast<PROJ_NETWORK_HANDLE *>(new File());
+    exchange->mapIdToHandle[openEvent->file_id] = handle;
+    return handle;
+}
+
+static void close_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle,
+                      void *user_data) {
+    auto exchange = static_cast<ExchangeWithCallback *>(user_data);
+    if (exchange->error)
+        return;
+    if (exchange->nextEvent >= exchange->events.size()) {
+        fprintf(stderr, "unexpected call to close()\n");
+        exchange->error = true;
+        return;
+    }
+    auto closeEvent =
+        dynamic_cast<CloseEvent *>(exchange->events[exchange->nextEvent].get());
+    if (!closeEvent) {
+        fprintf(stderr, "unexpected call to close(). "
+                        "Was expecting a %s event\n",
+                exchange->events[exchange->nextEvent]->type.c_str());
+        exchange->error = true;
+        return;
+    }
+    if (closeEvent->ctx != ctx) {
+        fprintf(stderr, "close() called with bad context\n");
+        exchange->error = true;
+        return;
+    }
+    if (exchange->mapIdToHandle[closeEvent->file_id] != handle) {
+        fprintf(stderr, "close() called with bad handle\n");
+        exchange->error = true;
+        return;
+    }
+    exchange->nextEvent++;
+    delete reinterpret_cast<File *>(handle);
+}
+
+static const char *get_header_value_cbk(PJ_CONTEXT * /* ctx */,
+                                        PROJ_NETWORK_HANDLE * /*handle*/,
+                                        const char * /*header_name*/,
+                                        void *user_data) {
+    auto exchange = static_cast<ExchangeWithCallback *>(user_data);
+    if (exchange->error)
+        return nullptr;
+    if (exchange->nextEvent >= exchange->events.size()) {
+        fprintf(stderr, "unexpected call to get_header_value()\n");
+        exchange->error = true;
+        return nullptr;
+    }
+    auto getHeaderValueEvent = dynamic_cast<GetHeaderValueEvent *>(
+        exchange->events[exchange->nextEvent].get());
+    if (!getHeaderValueEvent) {
+        fprintf(stderr, "unexpected call to get_header_value(). "
+                        "Was expecting a %s event\n",
+                exchange->events[exchange->nextEvent]->type.c_str());
+        exchange->error = true;
+        return nullptr;
+    }
+    exchange->nextEvent++;
+    return nullptr;
+}
+
+static unsigned long long get_file_size_cbk(PJ_CONTEXT * /* ctx */,
+                                            PROJ_NETWORK_HANDLE * /*handle*/,
+                                            void *user_data) {
+    auto exchange = static_cast<ExchangeWithCallback *>(user_data);
+    if (exchange->error)
+        return 0;
+    if (exchange->nextEvent >= exchange->events.size()) {
+        fprintf(stderr, "unexpected call to get_file_size()\n");
+        exchange->error = true;
+        return 0;
+    }
+    auto getFileSizeEvent = dynamic_cast<GetFileSizeEvent *>(
+        exchange->events[exchange->nextEvent].get());
+    if (!getFileSizeEvent) {
+        fprintf(stderr, "unexpected call to get_file_size(). "
+                        "Was expecting a %s event\n",
+                exchange->events[exchange->nextEvent]->type.c_str());
+        exchange->error = true;
+        return 0;
+    }
+    exchange->nextEvent++;
+    return 0;
+}
+
+static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle,
+                             unsigned long long offset, size_t size_to_read,
+                             void *buffer, void *user_data) {
+    auto exchange = static_cast<ExchangeWithCallback *>(user_data);
+    if (exchange->error)
+        return 0;
+    if (exchange->nextEvent >= exchange->events.size()) {
+        fprintf(stderr, "unexpected call to read_range(%ld, %ld)\n",
+                (long)offset, (long)size_to_read);
+        exchange->error = true;
+        return 0;
+    }
+    auto readRangeEvent = dynamic_cast<ReadRangeEvent *>(
+        exchange->events[exchange->nextEvent].get());
+    if (!readRangeEvent) {
+        fprintf(stderr, "unexpected call to read_range(). "
+                        "Was expecting a %s event\n",
+                exchange->events[exchange->nextEvent]->type.c_str());
+        exchange->error = true;
+        return 0;
+    }
+    if (exchange->mapIdToHandle[readRangeEvent->file_id] != handle) {
+        fprintf(stderr, "read_range() called with bad handle\n");
+        exchange->error = true;
+        return 0;
+    }
+    if (readRangeEvent->ctx != ctx || readRangeEvent->offset != offset ||
+        readRangeEvent->size_to_read != size_to_read) {
+        fprintf(stderr, "wrong call to read_range(%ld, %ld). Was expecting "
+                        "read_range(%ld, %ld)\n",
+                (long)offset, (long)size_to_read, (long)readRangeEvent->offset,
+                (long)readRangeEvent->size_to_read);
+        exchange->error = true;
+        return 0;
+    }
+    memcpy(buffer, readRangeEvent->response.data(),
+           readRangeEvent->response.size());
+    exchange->nextEvent++;
+    return readRangeEvent->response.size();
+}
+
+static const char *get_last_error_cbk(PJ_CONTEXT * /*ctx*/,
+                                      PROJ_NETWORK_HANDLE *, void *user_data) {
+    auto exchange = static_cast<ExchangeWithCallback *>(user_data);
+    if (exchange->error)
+        return "";
+    if (exchange->nextEvent >= exchange->events.size()) {
+        fprintf(stderr, "unexpected call to get_last_error()\n");
+        exchange->error = true;
+        return "";
+    }
+    auto getLastErrorEvent = dynamic_cast<GetLastErrorEvent *>(
+        exchange->events[exchange->nextEvent].get());
+    if (!getLastErrorEvent) {
+        fprintf(
+            stderr,
+            "unexpected call to get_last_error(). Was expecting a %s event\n",
+            exchange->events[exchange->nextEvent]->type.c_str());
+        exchange->error = true;
+        return "";
+    }
+    exchange->nextEvent++;
+    return "";
+}
+
+TEST(networking, custom) {
+    auto ctx = proj_context_create();
+    ExchangeWithCallback exchange;
+    ASSERT_TRUE(proj_context_set_network_callbacks(
+        ctx, open_cbk, close_cbk, get_header_value_cbk, get_file_size_cbk,
+        read_range_cbk, get_last_error_cbk, &exchange));
+
+    {
+        std::unique_ptr<OpenEvent> event(new OpenEvent());
+        event->ctx = ctx;
+        event->url = "https://foo/my.tif";
+        event->offset = 0;
+        event->size_to_read = 16384;
+        event->response.resize(16384);
+        event->file_id = 1;
+
+        const char *proj_source_data = getenv("PROJ_SOURCE_DATA");
+        ASSERT_TRUE(proj_source_data != nullptr);
+        std::string filename(proj_source_data);
+        filename += "/tests/egm96_15_uncompressed_truncated.tif";
+        FILE *f = fopen(filename.c_str(), "rb");
+        ASSERT_TRUE(f != nullptr);
+        ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U);
+        fclose(f);
+        exchange.events.emplace_back(std::move(event));
+    }
+    {
+        std::unique_ptr<CloseEvent> event(new CloseEvent());
+        event->ctx = ctx;
+        event->file_id = 1;
+        exchange.events.emplace_back(std::move(event));
+    }
+
+    auto P = proj_create(
+        ctx, "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1");
+
+    ASSERT_NE(P, nullptr);
+    ASSERT_TRUE(exchange.allConsumedAndNoError());
+
+    {
+        std::unique_ptr<OpenEvent> event(new OpenEvent());
+        event->ctx = ctx;
+        event->url = "https://foo/my.tif";
+        event->offset = 524288;
+        event->size_to_read = 278528;
+        event->response.resize(278528);
+        event->file_id = 2;
+        float f = 1.25;
+        for (size_t i = 0; i < 278528 / sizeof(float); i++) {
+            memcpy(&event->response[i * sizeof(float)], &f, sizeof(float));
+        }
+        exchange.events.emplace_back(std::move(event));
+    }
+
+    {
+        double lon = 2 / 180. * M_PI;
+        double lat = 49 / 180. * M_PI;
+        double z = 0;
+        ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat,
+                                     sizeof(double), 1, &z, sizeof(double), 1,
+                                     nullptr, 0, 0),
+                  1U);
+        EXPECT_EQ(z, 1.25);
+    }
+
+    ASSERT_TRUE(exchange.allConsumedAndNoError());
+
+    {
+        std::unique_ptr<ReadRangeEvent> event(new ReadRangeEvent());
+        event->ctx = ctx;
+        event->offset = 3670016;
+        event->size_to_read = 278528;
+        event->response.resize(278528);
+        event->file_id = 2;
+        float f = 2.25;
+        for (size_t i = 0; i < 278528 / sizeof(float); i++) {
+            memcpy(&event->response[i * sizeof(float)], &f, sizeof(float));
+        }
+        exchange.events.emplace_back(std::move(event));
+    }
+
+    {
+        double lon = 2 / 180. * M_PI;
+        double lat = -49 / 180. * M_PI;
+        double z = 0;
+        ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat,
+                                     sizeof(double), 1, &z, sizeof(double), 1,
+                                     nullptr, 0, 0),
+                  1U);
+        EXPECT_EQ(z, 2.25);
+    }
+    {
+        std::unique_ptr<CloseEvent> event(new CloseEvent());
+        event->ctx = ctx;
+        event->file_id = 2;
+        exchange.events.emplace_back(std::move(event));
+    }
+    proj_destroy(P);
+
+    ASSERT_TRUE(exchange.allConsumedAndNoError());
+
+    // Once again ! No network access
+
+    P = proj_create(ctx,
+                    "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1");
+    ASSERT_NE(P, nullptr);
+
+    {
+        double lon = 2 / 180. * M_PI;
+        double lat = 49 / 180. * M_PI;
+        double z = 0;
+        ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat,
+                                     sizeof(double), 1, &z, sizeof(double), 1,
+                                     nullptr, 0, 0),
+                  1U);
+        EXPECT_EQ(z, 1.25);
+    }
+
+    proj_destroy(P);
+
+    ASSERT_TRUE(exchange.allConsumedAndNoError());
+
+    proj_context_destroy(ctx);
+}
+
+} // namespace