Network: remove dedicated get_file_size() callback to use get_header_value(), and test

6 files changed, 162 insertions, 66 deletions

diff --git a/data/Makefile.am b/data/Makefile.am
index 59ab4e83..3778a2be 100644
--- a/data/Makefile.am
+++ b/data/Makefile.am
@@ -81,6 +81,7 @@ EXTRA_DIST = GL27 nad.lst nad27 nad83 \
 		tests/test_hgrid_with_subgrid_no_grid_name.tif \
 		tests/subset_of_gr3df97a.tif \
 		tests/egm96_15_uncompressed_truncated.tif \
+		tests/test_vgrid_single_strip_truncated.tif \
 		null \
 		generate_all_sql_in.cmake sql_filelist.cmake \
 		$(SQL_ORDERED_LIST)
diff --git a/data/tests/test_vgrid_single_strip_truncated.tif b/data/tests/test_vgrid_single_strip_truncated.tif
new file mode 100644
index 00000000..9a0030f6
--- /dev/null
+++ b/data/tests/test_vgrid_single_strip_truncated.tif
diff --git a/src/filemanager.cpp b/src/filemanager.cpp
index cc692616..ea0a63ea 100644
--- a/src/filemanager.cpp
+++ b/src/filemanager.cpp
@@ -266,6 +266,13 @@ void NetworkChunkCache::clear() { cache_.clear(); }
 
 static NetworkChunkCache gNetworkChunkCache{};
 
+struct FileProperties {
+    unsigned long long size;
+};
+
+static lru11::Cache<std::string, FileProperties, MyMutex>
+    gNetworkFileProperties{};
+
 // ---------------------------------------------------------------------------
 
 class NetworkFile : public File {
@@ -275,6 +282,7 @@ class NetworkFile : public File {
     unsigned long long m_pos = 0;
     size_t m_nBlocksToDownload = 1;
     unsigned long long m_lastDownloadedOffset;
+    unsigned long long m_filesize;
 
     NetworkFile(const NetworkFile &) = delete;
     NetworkFile &operator=(const NetworkFile &) = delete;
@@ -282,9 +290,10 @@ class NetworkFile : public File {
   protected:
     NetworkFile(PJ_CONTEXT *ctx, const std::string &url,
                 PROJ_NETWORK_HANDLE *handle,
-                unsigned long long lastDownloadOffset)
+                unsigned long long lastDownloadOffset,
+                unsigned long long filesize)
         : m_ctx(ctx), m_url(url), m_handle(handle),
-          m_lastDownloadedOffset(lastDownloadOffset) {}
+          m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {}
 
   public:
     ~NetworkFile() override;
@@ -300,9 +309,14 @@ class NetworkFile : public File {
 
 std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
     if (gNetworkChunkCache.get(filename, 0)) {
-        return std::unique_ptr<File>(
-            new NetworkFile(ctx, filename, nullptr,
-                            std::numeric_limits<unsigned long long>::max()));
+        unsigned long long filesize = 0;
+        FileProperties props;
+        if (gNetworkFileProperties.tryGet(filename, props)) {
+            filesize = props.size;
+        }
+        return std::unique_ptr<File>(new NetworkFile(
+            ctx, filename, nullptr,
+            std::numeric_limits<unsigned long long>::max(), filesize));
     } else {
         std::vector<unsigned char> buffer(DOWNLOAD_CHUNK_SIZE);
         size_t size_read = 0;
@@ -311,8 +325,24 @@ std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
                                  &size_read, ctx->networking.user_data);
         buffer.resize(size_read);
         gNetworkChunkCache.insert(filename, 0, std::move(buffer));
+
+        unsigned long long filesize = 0;
+        if (handle) {
+            const char *contentRange = ctx->networking.get_header_value(
+                ctx, handle, "Content-Range", ctx->networking.user_data);
+            if (contentRange) {
+                const char *slash = strchr(contentRange, '/');
+                if (slash) {
+                    filesize = std::stoull(slash + 1);
+                    FileProperties props;
+                    props.size = filesize;
+                    gNetworkFileProperties.insert(filename, props);
+                }
+            }
+        }
+
         return std::unique_ptr<File>(
-            handle ? new NetworkFile(ctx, filename, handle, size_read)
+            handle ? new NetworkFile(ctx, filename, handle, size_read, filesize)
                    : nullptr);
     }
 }
@@ -431,21 +461,7 @@ bool NetworkFile::seek(unsigned long long offset, int whence) {
     } else {
         if (offset != 0)
             return false;
-        if (!m_handle) {
-            size_t nRead = 0;
-            char dummy;
-            m_handle =
-                m_ctx->networking.open(m_ctx, m_url.c_str(), 0, 1, &dummy,
-                                       &nRead, m_ctx->networking.user_data);
-            if (!m_handle) {
-                return false;
-            }
-        }
-        const auto filesize = m_ctx->networking.get_file_size(
-            m_ctx, m_handle, m_ctx->networking.user_data);
-        if (filesize == 0)
-            return false;
-        m_pos = filesize;
+        m_pos = m_filesize;
     }
     return true;
 }
@@ -484,6 +500,7 @@ std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename) {
 struct CurlFileHandle {
     CURL *m_handle = nullptr;
     std::string m_headers;
+    std::string m_lastval{};
 
     CurlFileHandle(const CurlFileHandle &) = delete;
     CurlFileHandle &operator=(const CurlFileHandle &) = delete;
@@ -623,6 +640,27 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle,
     return std::min(size_to_read, body.size());
 }
 
+// ---------------------------------------------------------------------------
+
+static const char *pj_curl_get_header_value(PJ_CONTEXT *,
+                                            PROJ_NETWORK_HANDLE *raw_handle,
+                                            const char *header_name, void *) {
+    auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle);
+    auto pos = ci_find(handle->m_headers, header_name);
+    if (pos == std::string::npos)
+        return nullptr;
+    const char *c_str = handle->m_headers.c_str();
+    if (c_str[pos] == ':')
+        pos++;
+    while (c_str[pos] == ' ')
+        pos++;
+    auto posEnd = pos;
+    while (c_str[posEnd] != '\n' && c_str[posEnd] != '\0')
+        posEnd++;
+    handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos);
+    return handle->m_lastval.c_str();
+}
+
 #else
 
 // ---------------------------------------------------------------------------
@@ -660,6 +698,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) {
     ctx->networking.open = pj_curl_open;
     ctx->networking.close = pj_curl_close;
     ctx->networking.read_range = pj_curl_read_range;
+    ctx->networking.get_header_value = pj_curl_get_header_value;
 #else
     ctx->networking.open = no_op_network_open;
     ctx->networking.close = no_op_network_close;
@@ -669,7 +708,10 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) {
 
 // ---------------------------------------------------------------------------
 
-void FileManager::clearCache() { gNetworkChunkCache.clear(); }
+void FileManager::clearCache() {
+    gNetworkChunkCache.clear();
+    gNetworkFileProperties.clear();
+}
 
 // ---------------------------------------------------------------------------
 
@@ -687,7 +729,6 @@ NS_PROJ_END
  * @param open_cbk Callback to open a remote file given its URL
  * @param close_cbk Callback to close a remote file.
  * @param get_header_value_cbk Callback to get HTTP headers
- * @param get_file_size_cbk Callback to get the size of the remote file.
  * @param read_range_cbk Callback to read a range of bytes inside a remote file.
  * @param get_last_error_cbk Callback to get last error message.
  * @param user_data Arbitrary pointer provided by the user, and passed to the
@@ -698,20 +739,18 @@ int proj_context_set_network_callbacks(
     PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk,
     proj_network_close_cbk_type close_cbk,
     proj_network_get_header_value_cbk_type get_header_value_cbk,
-    proj_network_get_file_size_cbk_type get_file_size_cbk,
     proj_network_read_range_type read_range_cbk,
     proj_network_get_last_error_type get_last_error_cbk, void *user_data) {
     if (ctx == nullptr) {
         ctx = pj_get_default_ctx();
     }
-    if (!open_cbk || !close_cbk || !get_header_value_cbk ||
-        !get_file_size_cbk || !read_range_cbk || !get_last_error_cbk) {
+    if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk ||
+        !get_last_error_cbk) {
         return false;
     }
     ctx->networking.open = open_cbk;
     ctx->networking.close = close_cbk;
     ctx->networking.get_header_value = get_header_value_cbk;
-    ctx->networking.get_file_size = get_file_size_cbk;
     ctx->networking.read_range = read_range_cbk;
     ctx->networking.get_last_error = get_last_error_cbk;
     ctx->networking.user_data = user_data;
diff --git a/src/proj.h b/src/proj.h
index 5f080285..95bbacd1 100644
--- a/src/proj.h
+++ b/src/proj.h
@@ -367,8 +367,7 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE;
  * amount of bytes read (== size_to_read if the file is larger than size_to_read).
  * During this read, the implementation should make sure to store the HTTP
  * headers from the server response to be able to respond to
- * proj_network_get_header_value_cbk_type and proj_network_get_file_size_cbk_type
- * callbacks.
+ * proj_network_get_header_value_cbk_type callback.
  *
  * @return a non-NULL opaque handle in case of success.
  */
@@ -393,12 +392,6 @@ typedef const char* (*proj_network_get_header_value_cbk_type)(
                                             const char* header_name,
                                             void* user_data);
 
-/** Network access: get file size */
-typedef unsigned long long (*proj_network_get_file_size_cbk_type)(
-                                            PJ_CONTEXT* ctx,
-                                            PROJ_NETWORK_HANDLE* handle,
-                                            void* user_data);
-
 /** Network access: read range
  *
  * Read size_to_read bytes from handle, starting at offset, into
@@ -424,7 +417,6 @@ int PROJ_DLL proj_context_set_network_callbacks(
     proj_network_open_cbk_type open_cbk,
     proj_network_close_cbk_type close_cbk,
     proj_network_get_header_value_cbk_type get_header_value_cbk,
-    proj_network_get_file_size_cbk_type get_file_size_cbk,
     proj_network_read_range_type read_range_cbk,
     proj_network_get_last_error_type get_last_error_cbk,
     void* user_data);
diff --git a/src/proj_internal.h b/src/proj_internal.h
index aefeea39..0c148d02 100644
--- a/src/proj_internal.h
+++ b/src/proj_internal.h
@@ -671,7 +671,6 @@ struct projNetworkCallbacksAndData
     proj_network_open_cbk_type open = nullptr;
     proj_network_close_cbk_type close = nullptr;
     proj_network_get_header_value_cbk_type get_header_value = nullptr;
-    proj_network_get_file_size_cbk_type get_file_size = nullptr;
     proj_network_read_range_type read_range = nullptr;
     proj_network_get_last_error_type get_last_error = nullptr;
     void* user_data = nullptr;
diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp
index 6879e40d..5940814b 100644
--- a/test/unit/test_network.cpp
+++ b/test/unit/test_network.cpp
@@ -148,10 +148,10 @@ struct CloseEvent : public Event {
 
 struct GetHeaderValueEvent : public Event {
     GetHeaderValueEvent() { type = "GetHeaderValueEvent"; }
-};
 
-struct GetFileSizeEvent : public Event {
-    GetFileSizeEvent() { type = "GetFileSizeEvent"; }
+    int file_id = 0;
+    std::string key{};
+    std::string value{};
 };
 
 struct ReadRangeEvent : public Event {
@@ -255,9 +255,9 @@ static void close_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle,
     delete reinterpret_cast<File *>(handle);
 }
 
-static const char *get_header_value_cbk(PJ_CONTEXT * /* ctx */,
-                                        PROJ_NETWORK_HANDLE * /*handle*/,
-                                        const char * /*header_name*/,
+static const char *get_header_value_cbk(PJ_CONTEXT *ctx,
+                                        PROJ_NETWORK_HANDLE *handle,
+                                        const char *header_name,
                                         void *user_data) {
     auto exchange = static_cast<ExchangeWithCallback *>(user_data);
     if (exchange->error)
@@ -276,32 +276,25 @@ static const char *get_header_value_cbk(PJ_CONTEXT * /* ctx */,
         exchange->error = true;
         return nullptr;
     }
-    exchange->nextEvent++;
-    return nullptr;
-}
-
-static unsigned long long get_file_size_cbk(PJ_CONTEXT * /* ctx */,
-                                            PROJ_NETWORK_HANDLE * /*handle*/,
-                                            void *user_data) {
-    auto exchange = static_cast<ExchangeWithCallback *>(user_data);
-    if (exchange->error)
-        return 0;
-    if (exchange->nextEvent >= exchange->events.size()) {
-        fprintf(stderr, "unexpected call to get_file_size()\n");
+    if (getHeaderValueEvent->ctx != ctx) {
+        fprintf(stderr, "get_header_value() called with bad context\n");
         exchange->error = true;
-        return 0;
+        return nullptr;
     }
-    auto getFileSizeEvent = dynamic_cast<GetFileSizeEvent *>(
-        exchange->events[exchange->nextEvent].get());
-    if (!getFileSizeEvent) {
-        fprintf(stderr, "unexpected call to get_file_size(). "
-                        "Was expecting a %s event\n",
-                exchange->events[exchange->nextEvent]->type.c_str());
+    if (getHeaderValueEvent->key != header_name) {
+        fprintf(stderr, "wrong call to get_header_value(%s). Was expecting "
+                        "get_header_value(%s)\n",
+                header_name, getHeaderValueEvent->key.c_str());
         exchange->error = true;
-        return 0;
+        return nullptr;
+    }
+    if (exchange->mapIdToHandle[getHeaderValueEvent->file_id] != handle) {
+        fprintf(stderr, "get_header_value() called with bad handle\n");
+        exchange->error = true;
+        return nullptr;
     }
     exchange->nextEvent++;
-    return 0;
+    return getHeaderValueEvent->value.c_str();
 }
 
 static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle,
@@ -373,8 +366,8 @@ TEST(networking, custom) {
     auto ctx = proj_context_create();
     ExchangeWithCallback exchange;
     ASSERT_TRUE(proj_context_set_network_callbacks(
-        ctx, open_cbk, close_cbk, get_header_value_cbk, get_file_size_cbk,
-        read_range_cbk, get_last_error_cbk, &exchange));
+        ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk,
+        get_last_error_cbk, &exchange));
 
     {
         std::unique_ptr<OpenEvent> event(new OpenEvent());
@@ -396,6 +389,14 @@ TEST(networking, custom) {
         exchange.events.emplace_back(std::move(event));
     }
     {
+        std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent());
+        event->ctx = ctx;
+        event->key = "Content-Range";
+        event->value = "dummy"; // dummy value: not used
+        event->file_id = 1;
+        exchange.events.emplace_back(std::move(event));
+    }
+    {
         std::unique_ptr<CloseEvent> event(new CloseEvent());
         event->ctx = ctx;
         event->file_id = 1;
@@ -494,4 +495,68 @@ TEST(networking, custom) {
     proj_context_destroy(ctx);
 }
 
+// ---------------------------------------------------------------------------
+
+TEST(networking, getfilesize) {
+    auto ctx = proj_context_create();
+    ExchangeWithCallback exchange;
+    ASSERT_TRUE(proj_context_set_network_callbacks(
+        ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk,
+        get_last_error_cbk, &exchange));
+
+    {
+        std::unique_ptr<OpenEvent> event(new OpenEvent());
+        event->ctx = ctx;
+        event->url = "https://foo/getfilesize.tif";
+        event->offset = 0;
+        event->size_to_read = 16384;
+        event->response.resize(16384);
+        event->file_id = 1;
+
+        const char *proj_source_data = getenv("PROJ_SOURCE_DATA");
+        ASSERT_TRUE(proj_source_data != nullptr);
+        std::string filename(proj_source_data);
+        filename += "/tests/test_vgrid_single_strip_truncated.tif";
+        FILE *f = fopen(filename.c_str(), "rb");
+        ASSERT_TRUE(f != nullptr);
+        ASSERT_EQ(fread(&event->response[0], 1, 550, f), 550U);
+        fclose(f);
+        exchange.events.emplace_back(std::move(event));
+    }
+    {
+        std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent());
+        event->ctx = ctx;
+        event->key = "Content-Range";
+        event->value = "bytes 0-16383/4153510";
+        event->file_id = 1;
+        exchange.events.emplace_back(std::move(event));
+    }
+    {
+        std::unique_ptr<CloseEvent> event(new CloseEvent());
+        event->ctx = ctx;
+        event->file_id = 1;
+        exchange.events.emplace_back(std::move(event));
+    }
+
+    auto P = proj_create(
+        ctx,
+        "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1");
+
+    ASSERT_NE(P, nullptr);
+    ASSERT_TRUE(exchange.allConsumedAndNoError());
+
+    proj_destroy(P);
+
+    P = proj_create(
+        ctx,
+        "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1");
+
+    ASSERT_NE(P, nullptr);
+    ASSERT_TRUE(exchange.allConsumedAndNoError());
+
+    proj_destroy(P);
+
+    proj_context_destroy(ctx);
+}
+
 } // namespace