Merge pull request #1813 from rouault/rfc4_network

[RFC4_dev] Add networking capabilities

1 files changed, 997 insertions, 0 deletions

diff --git a/src/filemanager.cpp b/src/filemanager.cpp
new file mode 100644
index 00000000..625dca03
--- /dev/null
+++ b/src/filemanager.cpp
@@ -0,0 +1,997 @@
+/******************************************************************************
+ * Project:  PROJ
+ * Purpose:  File manager
+ * Author:   Even Rouault, <even.rouault at spatialys.com>
+ *
+ ******************************************************************************
+ * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *****************************************************************************/
+
+#ifndef FROM_PROJ_CPP
+#define FROM_PROJ_CPP
+#endif
+#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS
+
+#include <algorithm>
+#include <functional>
+#include <limits>
+
+#include "filemanager.hpp"
+#include "proj.h"
+#include "proj/internal/internal.hpp"
+#include "proj/internal/lru_cache.hpp"
+#include "proj_internal.h"
+
+#ifdef __MINGW32__
+// mingw32-win32 doesn't implement std::mutex
+namespace {
+class MyMutex {
+  public:
+    // cppcheck-suppress functionStatic
+    void lock() { pj_acquire_lock(); }
+    // cppcheck-suppress functionStatic
+    void unlock() { pj_release_lock(); }
+};
+}
+#else
+#include <mutex>
+#define MyMutex std::mutex
+#endif
+
+#ifdef CURL_ENABLED
+#include <curl/curl.h>
+#include <sqlite3.h> // for sqlite3_snprintf
+#endif
+
+#if defined(__linux)
+#include <unistd.h>
+#elif defined(_WIN32)
+#include <windows.h>
+#elif defined(__MACH__) && defined(__APPLE__)
+#include <mach-o/dyld.h>
+#elif defined(__FreeBSD__)
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#endif
+
+//! @cond Doxygen_Suppress
+
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+
+using namespace NS_PROJ::internal;
+
+NS_PROJ_START
+
+// ---------------------------------------------------------------------------
+
+File::File(const std::string &name) : name_(name) {}
+
+// ---------------------------------------------------------------------------
+
+File::~File() = default;
+
+// ---------------------------------------------------------------------------
+
+class FileStdio : public File {
+    PJ_CONTEXT *m_ctx;
+    FILE *m_fp;
+
+    FileStdio(const FileStdio &) = delete;
+    FileStdio &operator=(const FileStdio &) = delete;
+
+  protected:
+    FileStdio(const std::string &name, PJ_CONTEXT *ctx, FILE *fp)
+        : File(name), m_ctx(ctx), m_fp(fp) {}
+
+  public:
+    ~FileStdio() override;
+
+    size_t read(void *buffer, size_t sizeBytes) override;
+    bool seek(unsigned long long offset, int whence = SEEK_SET) override;
+    unsigned long long tell() override;
+    void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; }
+
+    static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename);
+};
+
+// ---------------------------------------------------------------------------
+
+FileStdio::~FileStdio() { fclose(m_fp); }
+
+// ---------------------------------------------------------------------------
+
+size_t FileStdio::read(void *buffer, size_t sizeBytes) {
+    return fread(buffer, 1, sizeBytes, m_fp);
+}
+
+// ---------------------------------------------------------------------------
+
+bool FileStdio::seek(unsigned long long offset, int whence) {
+    // TODO one day: use 64-bit offset compatible API
+    if (offset != static_cast<unsigned long long>(static_cast<long>(offset))) {
+        pj_log(m_ctx, PJ_LOG_ERROR,
+               "Attempt at seeking to a 64 bit offset. Not supported yet");
+        return false;
+    }
+    return fseek(m_fp, static_cast<long>(offset), whence) == 0;
+}
+
+// ---------------------------------------------------------------------------
+
+unsigned long long FileStdio::tell() {
+    // TODO one day: use 64-bit offset compatible API
+    return ftell(m_fp);
+}
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<File> FileStdio::open(PJ_CONTEXT *ctx, const char *filename) {
+    auto fp = fopen(filename, "rb");
+    return std::unique_ptr<File>(fp ? new FileStdio(filename, ctx, fp)
+                                    : nullptr);
+}
+
+// ---------------------------------------------------------------------------
+
+#ifndef REMOVE_LEGACY_SUPPORT
+
+class FileLegacyAdapter : public File {
+    PJ_CONTEXT *m_ctx;
+    PAFile m_fp;
+
+    FileLegacyAdapter(const FileLegacyAdapter &) = delete;
+    FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete;
+
+  protected:
+    FileLegacyAdapter(const std::string &name, PJ_CONTEXT *ctx, PAFile fp)
+        : File(name), m_ctx(ctx), m_fp(fp) {}
+
+  public:
+    ~FileLegacyAdapter() override;
+
+    size_t read(void *buffer, size_t sizeBytes) override;
+    bool seek(unsigned long long offset, int whence = SEEK_SET) override;
+    unsigned long long tell() override;
+    void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; }
+
+    static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename);
+};
+
+// ---------------------------------------------------------------------------
+
+FileLegacyAdapter::~FileLegacyAdapter() { pj_ctx_fclose(m_ctx, m_fp); }
+
+// ---------------------------------------------------------------------------
+
+size_t FileLegacyAdapter::read(void *buffer, size_t sizeBytes) {
+    return pj_ctx_fread(m_ctx, buffer, 1, sizeBytes, m_fp);
+}
+
+// ---------------------------------------------------------------------------
+
+bool FileLegacyAdapter::seek(unsigned long long offset, int whence) {
+    if (offset != static_cast<unsigned long long>(static_cast<long>(offset))) {
+        pj_log(m_ctx, PJ_LOG_ERROR,
+               "Attempt at seeking to a 64 bit offset. Not supported yet");
+        return false;
+    }
+    return pj_ctx_fseek(m_ctx, m_fp, static_cast<long>(offset), whence) == 0;
+}
+
+// ---------------------------------------------------------------------------
+
+unsigned long long FileLegacyAdapter::tell() {
+    return pj_ctx_ftell(m_ctx, m_fp);
+}
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<File> FileLegacyAdapter::open(PJ_CONTEXT *ctx,
+                                              const char *filename) {
+    auto fid = pj_ctx_fopen(ctx, filename, "rb");
+    return std::unique_ptr<File>(fid ? new FileLegacyAdapter(filename, ctx, fid)
+                                     : nullptr);
+}
+
+#endif // REMOVE_LEGACY_SUPPORT
+
+// ---------------------------------------------------------------------------
+
+constexpr size_t DOWNLOAD_CHUNK_SIZE = 16 * 1024;
+constexpr int MAX_CHUNKS = 64;
+
+class NetworkChunkCache {
+  public:
+    void insert(const std::string &url, unsigned long long chunkIdx,
+                std::vector<unsigned char> &&data);
+
+    std::shared_ptr<std::vector<unsigned char>>
+    get(const std::string &url, unsigned long long chunkIdx);
+
+    void clear();
+
+  private:
+    struct Key {
+        std::string url;
+        unsigned long long chunkIdx;
+
+        Key(const std::string &urlIn, unsigned long long chunkIdxIn)
+            : url(urlIn), chunkIdx(chunkIdxIn) {}
+        bool operator==(const Key &other) const {
+            return url == other.url && chunkIdx == other.chunkIdx;
+        }
+    };
+
+    struct KeyHasher {
+        std::size_t operator()(const Key &k) const {
+            return std::hash<std::string>{}(k.url) ^
+                   (std::hash<unsigned long long>{}(k.chunkIdx) << 1);
+        }
+    };
+
+    lru11::Cache<
+        Key, std::shared_ptr<std::vector<unsigned char>>, MyMutex,
+        std::unordered_map<
+            Key,
+            typename std::list<lru11::KeyValuePair<
+                Key, std::shared_ptr<std::vector<unsigned char>>>>::iterator,
+            KeyHasher>>
+        cache_{MAX_CHUNKS};
+};
+
+// ---------------------------------------------------------------------------
+
+void NetworkChunkCache::insert(const std::string &url,
+                               unsigned long long chunkIdx,
+                               std::vector<unsigned char> &&data) {
+    cache_.insert(
+        Key(url, chunkIdx),
+        std::make_shared<std::vector<unsigned char>>(std::move(data)));
+}
+
+// ---------------------------------------------------------------------------
+
+std::shared_ptr<std::vector<unsigned char>>
+NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) {
+    std::shared_ptr<std::vector<unsigned char>> ret;
+    cache_.tryGet(Key(url, chunkIdx), ret);
+    return ret;
+}
+
+// ---------------------------------------------------------------------------
+
+void NetworkChunkCache::clear() { cache_.clear(); }
+
+// ---------------------------------------------------------------------------
+
+static NetworkChunkCache gNetworkChunkCache{};
+
+struct FileProperties {
+    unsigned long long size;
+};
+
+static lru11::Cache<std::string, FileProperties, MyMutex>
+    gNetworkFileProperties{};
+
+// ---------------------------------------------------------------------------
+
+class NetworkFile : public File {
+    PJ_CONTEXT *m_ctx;
+    std::string m_url;
+    PROJ_NETWORK_HANDLE *m_handle;
+    unsigned long long m_pos = 0;
+    size_t m_nBlocksToDownload = 1;
+    unsigned long long m_lastDownloadedOffset;
+    unsigned long long m_filesize;
+    proj_network_close_cbk_type m_closeCbk;
+
+    NetworkFile(const NetworkFile &) = delete;
+    NetworkFile &operator=(const NetworkFile &) = delete;
+
+  protected:
+    NetworkFile(PJ_CONTEXT *ctx, const std::string &url,
+                PROJ_NETWORK_HANDLE *handle,
+                unsigned long long lastDownloadOffset,
+                unsigned long long filesize)
+        : File(url), m_ctx(ctx), m_url(url), m_handle(handle),
+          m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize),
+          m_closeCbk(ctx->networking.close) {}
+
+  public:
+    ~NetworkFile() override;
+
+    size_t read(void *buffer, size_t sizeBytes) override;
+    bool seek(unsigned long long offset, int whence) override;
+    unsigned long long tell() override;
+    void reassign_context(PJ_CONTEXT *ctx) override;
+
+    static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename);
+};
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) {
+    if (gNetworkChunkCache.get(filename, 0)) {
+        unsigned long long filesize = 0;
+        FileProperties props;
+        if (gNetworkFileProperties.tryGet(filename, props)) {
+            filesize = props.size;
+        }
+        return std::unique_ptr<File>(new NetworkFile(
+            ctx, filename, nullptr,
+            std::numeric_limits<unsigned long long>::max(), filesize));
+    } else {
+        std::vector<unsigned char> buffer(DOWNLOAD_CHUNK_SIZE);
+        size_t size_read = 0;
+        std::string errorBuffer;
+        errorBuffer.resize(1024);
+
+        auto handle = ctx->networking.open(
+            ctx, filename, 0, buffer.size(), &buffer[0], &size_read,
+            errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data);
+        buffer.resize(size_read);
+        gNetworkChunkCache.insert(filename, 0, std::move(buffer));
+        if (!handle) {
+            errorBuffer.resize(strlen(errorBuffer.data()));
+            pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename,
+                   errorBuffer.c_str());
+        }
+
+        unsigned long long filesize = 0;
+        if (handle) {
+            const char *contentRange = ctx->networking.get_header_value(
+                ctx, handle, "Content-Range", ctx->networking.user_data);
+            if (contentRange) {
+                const char *slash = strchr(contentRange, '/');
+                if (slash) {
+                    filesize = std::stoull(slash + 1);
+                    FileProperties props;
+                    props.size = filesize;
+                    gNetworkFileProperties.insert(filename, props);
+                }
+            }
+        }
+
+        return std::unique_ptr<File>(
+            handle ? new NetworkFile(ctx, filename, handle, size_read, filesize)
+                   : nullptr);
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+size_t NetworkFile::read(void *buffer, size_t sizeBytes) {
+
+    if (sizeBytes == 0)
+        return 0;
+
+    auto iterOffset = m_pos;
+    while (sizeBytes) {
+        const auto chunkIdxToDownload = iterOffset / DOWNLOAD_CHUNK_SIZE;
+        const auto offsetToDownload = chunkIdxToDownload * DOWNLOAD_CHUNK_SIZE;
+        std::vector<unsigned char> region;
+        auto pChunk = gNetworkChunkCache.get(m_url, chunkIdxToDownload);
+        if (pChunk != nullptr) {
+            region = *pChunk;
+        } else {
+            if (offsetToDownload == m_lastDownloadedOffset) {
+                // In case of consecutive reads (of small size), we use a
+                // heuristic that we will read the file sequentially, so
+                // we double the requested size to decrease the number of
+                // client/server roundtrips.
+                if (m_nBlocksToDownload < 100)
+                    m_nBlocksToDownload *= 2;
+            } else {
+                // Random reads. Cancel the above heuristics.
+                m_nBlocksToDownload = 1;
+            }
+
+            // Ensure that we will request at least the number of blocks
+            // to satisfy the remaining buffer size to read.
+            const auto endOffsetToDownload =
+                ((iterOffset + sizeBytes + DOWNLOAD_CHUNK_SIZE - 1) /
+                 DOWNLOAD_CHUNK_SIZE) *
+                DOWNLOAD_CHUNK_SIZE;
+            const auto nMinBlocksToDownload = static_cast<size_t>(
+                (endOffsetToDownload - offsetToDownload) / DOWNLOAD_CHUNK_SIZE);
+            if (m_nBlocksToDownload < nMinBlocksToDownload)
+                m_nBlocksToDownload = nMinBlocksToDownload;
+
+            // Avoid reading already cached data.
+            // Note: this might get evicted if concurrent reads are done, but
+            // this should not cause bugs. Just missed optimization.
+            for (size_t i = 1; i < m_nBlocksToDownload; i++) {
+                if (gNetworkChunkCache.get(m_url, chunkIdxToDownload + i) !=
+                    nullptr) {
+                    m_nBlocksToDownload = i;
+                    break;
+                }
+            }
+
+            if (m_nBlocksToDownload > MAX_CHUNKS)
+                m_nBlocksToDownload = MAX_CHUNKS;
+
+            region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE);
+            size_t nRead = 0;
+            std::string errorBuffer;
+            errorBuffer.resize(1024);
+            if (!m_handle) {
+                m_handle = m_ctx->networking.open(
+                    m_ctx, m_url.c_str(), offsetToDownload,
+                    m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, &region[0],
+                    &nRead, errorBuffer.size(), &errorBuffer[0],
+                    m_ctx->networking.user_data);
+                if (!m_handle) {
+                    return 0;
+                }
+            } else {
+                nRead = m_ctx->networking.read_range(
+                    m_ctx, m_handle, offsetToDownload,
+                    m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, &region[0],
+                    errorBuffer.size(), &errorBuffer[0],
+                    m_ctx->networking.user_data);
+            }
+            if (nRead == 0) {
+                errorBuffer.resize(strlen(errorBuffer.data()));
+                if (!errorBuffer.empty()) {
+                    pj_log(m_ctx, PJ_LOG_ERROR, "Cannot read in %s: %s",
+                           m_url.c_str(), errorBuffer.c_str());
+                }
+                return 0;
+            }
+            region.resize(nRead);
+            m_lastDownloadedOffset = offsetToDownload + nRead;
+
+            const auto nChunks =
+                (region.size() + DOWNLOAD_CHUNK_SIZE - 1) / DOWNLOAD_CHUNK_SIZE;
+            for (size_t i = 0; i < nChunks; i++) {
+                std::vector<unsigned char> chunk(
+                    region.data() + i * DOWNLOAD_CHUNK_SIZE,
+                    region.data() +
+                        std::min((i + 1) * DOWNLOAD_CHUNK_SIZE, region.size()));
+                gNetworkChunkCache.insert(m_url, chunkIdxToDownload + i,
+                                          std::move(chunk));
+            }
+        }
+        const size_t nToCopy = static_cast<size_t>(
+            std::min(static_cast<unsigned long long>(sizeBytes),
+                     region.size() - (iterOffset - offsetToDownload)));
+        memcpy(buffer, region.data() + iterOffset - offsetToDownload, nToCopy);
+        buffer = static_cast<char *>(buffer) + nToCopy;
+        iterOffset += nToCopy;
+        sizeBytes -= nToCopy;
+        if (region.size() < static_cast<size_t>(DOWNLOAD_CHUNK_SIZE) &&
+            sizeBytes != 0) {
+            break;
+        }
+    }
+
+    size_t nRead = static_cast<size_t>(iterOffset - m_pos);
+    m_pos = iterOffset;
+    return nRead;
+}
+
+// ---------------------------------------------------------------------------
+
+bool NetworkFile::seek(unsigned long long offset, int whence) {
+    if (whence == SEEK_SET) {
+        m_pos = offset;
+    } else if (whence == SEEK_CUR) {
+        m_pos += offset;
+    } else {
+        if (offset != 0)
+            return false;
+        m_pos = m_filesize;
+    }
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+unsigned long long NetworkFile::tell() { return m_pos; }
+
+// ---------------------------------------------------------------------------
+
+NetworkFile::~NetworkFile() {
+    if (m_handle) {
+        m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data);
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+void NetworkFile::reassign_context(PJ_CONTEXT *ctx) {
+    m_ctx = ctx;
+    if (m_closeCbk != m_ctx->networking.close) {
+        pj_log(m_ctx, PJ_LOG_ERROR,
+               "Networking close callback has changed following context "
+               "reassignment ! This is highly suspicious");
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename) {
+#ifndef REMOVE_LEGACY_SUPPORT
+    // If the user has specified a legacy fileapi, use it
+    if (ctx->fileapi_legacy != pj_get_default_fileapi()) {
+        return FileLegacyAdapter::open(ctx, filename);
+    }
+#endif
+    if (starts_with(filename, "http://") || starts_with(filename, "https://")) {
+        if (!pj_context_is_network_enabled(ctx)) {
+            pj_log(
+                ctx, PJ_LOG_ERROR,
+                "Attempt at accessing remote resource not authorized. Either "
+                "set PROJ_NETWORK=ON or "
+                "proj_context_set_enable_network(ctx, TRUE)");
+            return nullptr;
+        }
+        return NetworkFile::open(ctx, filename);
+    }
+    return FileStdio::open(ctx, filename);
+}
+
+// ---------------------------------------------------------------------------
+
+#ifdef CURL_ENABLED
+
+struct CurlFileHandle {
+    std::string m_url;
+    CURL *m_handle;
+    std::string m_headers{};
+    std::string m_lastval{};
+    std::string m_useragent{};
+    char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
+
+    CurlFileHandle(const CurlFileHandle &) = delete;
+    CurlFileHandle &operator=(const CurlFileHandle &) = delete;
+
+    explicit CurlFileHandle(const char *url, CURL *handle);
+    ~CurlFileHandle();
+
+    static PROJ_NETWORK_HANDLE *
+    open(PJ_CONTEXT *, const char *url, unsigned long long offset,
+         size_t size_to_read, void *buffer, size_t *out_size_read,
+         size_t error_string_max_size, char *out_error_string, void *);
+};
+
+// ---------------------------------------------------------------------------
+
+static std::string GetExecutableName() {
+#if defined(__linux)
+    std::string path;
+    path.resize(1024);
+    const auto ret = readlink("/proc/self/exe", &path[0], path.size());
+    if (ret > 0) {
+        path.resize(ret);
+        const auto pos = path.rfind('/');
+        if (pos != std::string::npos) {
+            path = path.substr(pos + 1);
+        }
+        return path;
+    }
+#elif defined(_WIN32)
+    std::string path;
+    path.resize(1024);
+    if (GetModuleFileNameA(nullptr, &path[0],
+                           static_cast<DWORD>(path.size()))) {
+        path.resize(strlen(path.c_str()));
+        const auto pos = path.rfind('\\');
+        if (pos != std::string::npos) {
+            path = path.substr(pos + 1);
+        }
+        return path;
+    }
+#elif defined(__MACH__) && defined(__APPLE__)
+    std::string path;
+    path.resize(1024);
+    uint32_t size = static_cast<uint32_t>(path.size());
+    if (_NSGetExecutablePath(&path[0], &size) == 0) {
+        path.resize(strlen(path.c_str()));
+        const auto pos = path.rfind('/');
+        if (pos != std::string::npos) {
+            path = path.substr(pos + 1);
+        }
+        return path;
+    }
+#elif defined(__FreeBSD__)
+    int mib[4];
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_PROC;
+    mib[2] = KERN_PROC_PATHNAME;
+    mib[3] = -1;
+    std::string path;
+    path.resize(1024);
+    size_t size = path.size();
+    if (sysctl(mib, 4, &path[0], &size, nullptr, 0) == 0) {
+        path.resize(strlen(path.c_str()));
+        const auto pos = path.rfind('/');
+        if (pos != std::string::npos) {
+            path = path.substr(pos + 1);
+        }
+        return path;
+    }
+#endif
+
+    return std::string();
+}
+
+// ---------------------------------------------------------------------------
+
+CurlFileHandle::CurlFileHandle(const char *url, CURL *handle)
+    : m_url(url), m_handle(handle) {
+    curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str());
+
+    if (getenv("PROJ_CURL_VERBOSE"))
+        curl_easy_setopt(handle, CURLOPT_VERBOSE, 1);
+
+// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer.
+#if LIBCURL_VERSION_NUM >= 0x073600
+    curl_easy_setopt(handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L);
+#endif
+
+    // Enable following redirections.  Requires libcurl 7.10.1 at least.
+    curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1);
+    curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10);
+
+    if (getenv("PROJ_UNSAFE_SSL")) {
+        curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L);
+        curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L);
+    }
+
+    curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf);
+
+    if (getenv("PROJ_NO_USERAGENT") == nullptr) {
+        m_useragent = "PROJ " STR(PROJ_VERSION_MAJOR) "." STR(
+            PROJ_VERSION_MINOR) "." STR(PROJ_VERSION_PATCH);
+        const auto exeName = GetExecutableName();
+        if (!exeName.empty()) {
+            m_useragent = exeName + " using " + m_useragent;
+        }
+        curl_easy_setopt(handle, CURLOPT_USERAGENT, m_useragent.data());
+    }
+}
+
+// ---------------------------------------------------------------------------
+
+CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); }
+
+// ---------------------------------------------------------------------------
+
+static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb,
+                                 void *req) {
+    const size_t nSize = count * nmemb;
+    auto pStr = static_cast<std::string *>(req);
+    if (pStr->size() + nSize > pStr->capacity()) {
+        // to avoid servers not honouring Range to cause excessive memory
+        // allocation
+        return 0;
+    }
+    pStr->append(static_cast<const char *>(buffer), nSize);
+    return nmemb;
+}
+
+// ---------------------------------------------------------------------------
+
+PROJ_NETWORK_HANDLE *CurlFileHandle::open(PJ_CONTEXT *, const char *url,
+                                          unsigned long long offset,
+                                          size_t size_to_read, void *buffer,
+                                          size_t *out_size_read,
+                                          size_t error_string_max_size,
+                                          char *out_error_string, void *) {
+    CURL *hCurlHandle = curl_easy_init();
+    if (!hCurlHandle)
+        return nullptr;
+
+    auto file =
+        std::unique_ptr<CurlFileHandle>(new CurlFileHandle(url, hCurlHandle));
+
+    char szBuffer[128];
+    sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset,
+                     offset + size_to_read - 1);
+    curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer);
+
+    std::string headers;
+    headers.reserve(16 * 1024);
+    curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers);
+    curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, pj_curl_write_func);
+
+    std::string body;
+    body.reserve(size_to_read);
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body);
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func);
+
+    file->m_szCurlErrBuf[0] = '\0';
+
+    curl_easy_perform(hCurlHandle);
+
+    long response_code = 0;
+    curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
+
+    curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, nullptr);
+    curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, nullptr);
+
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr);
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr);
+
+    if (response_code == 0 || response_code >= 300) {
+        if (out_error_string) {
+            if (file->m_szCurlErrBuf[0]) {
+                snprintf(out_error_string, error_string_max_size, "%s",
+                         file->m_szCurlErrBuf);
+            } else {
+                snprintf(out_error_string, error_string_max_size,
+                         "HTTP error %ld: %s", response_code, body.c_str());
+            }
+        }
+        return nullptr;
+    }
+    if (out_error_string && error_string_max_size) {
+        out_error_string[0] = '\0';
+    }
+
+    if (!body.empty()) {
+        memcpy(buffer, body.data(), std::min(size_to_read, body.size()));
+    }
+    *out_size_read = std::min(size_to_read, body.size());
+
+    file->m_headers = std::move(headers);
+    return reinterpret_cast<PROJ_NETWORK_HANDLE *>(file.release());
+}
+
+// ---------------------------------------------------------------------------
+
+static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle,
+                          void * /*user_data*/) {
+    delete reinterpret_cast<CurlFileHandle *>(handle);
+}
+
+// ---------------------------------------------------------------------------
+
+static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle,
+                                 unsigned long long offset, size_t size_to_read,
+                                 void *buffer, size_t error_string_max_size,
+                                 char *out_error_string, void *) {
+    auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle);
+    auto hCurlHandle = handle->m_handle;
+
+    char szBuffer[128];
+    sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset,
+                     offset + size_to_read - 1);
+    curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer);
+
+    std::string body;
+    body.reserve(size_to_read);
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body);
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func);
+
+    handle->m_szCurlErrBuf[0] = '\0';
+
+    curl_easy_perform(hCurlHandle);
+
+    long response_code = 0;
+    curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
+
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr);
+    curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr);
+
+    if (response_code == 0 || response_code >= 300) {
+        if (out_error_string) {
+            if (handle->m_szCurlErrBuf[0]) {
+                snprintf(out_error_string, error_string_max_size, "%s",
+                         handle->m_szCurlErrBuf);
+            } else {
+                snprintf(out_error_string, error_string_max_size,
+                         "HTTP error %ld: %s", response_code, body.c_str());
+            }
+        }
+        return 0;
+    }
+    if (out_error_string && error_string_max_size) {
+        out_error_string[0] = '\0';
+    }
+
+    if (!body.empty()) {
+        memcpy(buffer, body.data(), std::min(size_to_read, body.size()));
+    }
+    return std::min(size_to_read, body.size());
+}
+
+// ---------------------------------------------------------------------------
+
+static const char *pj_curl_get_header_value(PJ_CONTEXT *,
+                                            PROJ_NETWORK_HANDLE *raw_handle,
+                                            const char *header_name, void *) {
+    auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle);
+    auto pos = ci_find(handle->m_headers, header_name);
+    if (pos == std::string::npos)
+        return nullptr;
+    const char *c_str = handle->m_headers.c_str();
+    if (c_str[pos] == ':')
+        pos++;
+    while (c_str[pos] == ' ')
+        pos++;
+    auto posEnd = pos;
+    while (c_str[posEnd] != '\n' && c_str[posEnd] != '\0')
+        posEnd++;
+    handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos);
+    return handle->m_lastval.c_str();
+}
+
+#else
+
+// ---------------------------------------------------------------------------
+
+static PROJ_NETWORK_HANDLE *
+no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */,
+                   unsigned long long, /* offset */
+                   size_t,             /* size to read */
+                   void *,             /* buffer to update with bytes read*/
+                   size_t *,           /* output: size actually read */
+                   size_t error_string_max_size, char *out_error_string,
+                   void * /*user_data*/) {
+    if (out_error_string) {
+        snprintf(out_error_string, error_string_max_size, "%s",
+                 "Network functionality not available");
+    }
+    return nullptr;
+}
+
+// ---------------------------------------------------------------------------
+
+static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *,
+                                void * /*user_data*/) {}
+
+#endif
+
+// ---------------------------------------------------------------------------
+
+void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) {
+#ifdef CURL_ENABLED
+    ctx->networking.open = CurlFileHandle::open;
+    ctx->networking.close = pj_curl_close;
+    ctx->networking.read_range = pj_curl_read_range;
+    ctx->networking.get_header_value = pj_curl_get_header_value;
+#else
+    ctx->networking.open = no_op_network_open;
+    ctx->networking.close = no_op_network_close;
+#endif
+}
+
+// ---------------------------------------------------------------------------
+
+void FileManager::clearCache() {
+    gNetworkChunkCache.clear();
+    gNetworkFileProperties.clear();
+}
+
+// ---------------------------------------------------------------------------
+
+NS_PROJ_END
+
+//! @endcond
+
+// ---------------------------------------------------------------------------
+
+/** Define a custom set of callbacks for network access.
+ *
+ * All callbacks should be provided (non NULL pointers).
+ *
+ * @param ctx PROJ context, or NULL
+ * @param open_cbk Callback to open a remote file given its URL
+ * @param close_cbk Callback to close a remote file.
+ * @param get_header_value_cbk Callback to get HTTP headers
+ * @param read_range_cbk Callback to read a range of bytes inside a remote file.
+ * @param user_data Arbitrary pointer provided by the user, and passed to the
+ * above callbacks. May be NULL.
+ * @return TRUE in case of success.
+ */
+int proj_context_set_network_callbacks(
+    PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk,
+    proj_network_close_cbk_type close_cbk,
+    proj_network_get_header_value_cbk_type get_header_value_cbk,
+    proj_network_read_range_type read_range_cbk, void *user_data) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk) {
+        return false;
+    }
+    ctx->networking.open = open_cbk;
+    ctx->networking.close = close_cbk;
+    ctx->networking.get_header_value = get_header_value_cbk;
+    ctx->networking.read_range = read_range_cbk;
+    ctx->networking.user_data = user_data;
+    return true;
+}
+
+// ---------------------------------------------------------------------------
+
+/** Enable or disable network access.
+*
+* This overrides the default endpoint in the PROJ configuration file or with
+* the PROJ_NETWORK environment variable.
+*
+* @param ctx PROJ context, or NULL
+* @param enable TRUE if network access is allowed.
+* @return TRUE if network access is possible. That is either libcurl is
+*         available, or an alternate interface has been set.
+*/
+int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    // Load ini file, now so as to override its network settings
+    pj_load_ini(ctx);
+    ctx->networking.enabled_env_variable_checked = true;
+    ctx->networking.enabled = enable != FALSE;
+#ifdef CURL_ENABLED
+    return ctx->networking.enabled;
+#else
+    return ctx->networking.enabled &&
+           ctx->networking.open != NS_PROJ::no_op_network_open;
+#endif
+}
+
+// ---------------------------------------------------------------------------
+
+/** Define the URL endpoint to query for remote grids.
+*
+* This overrides the default endpoint in the PROJ configuration file or with
+* the PROJ_NETWORK_ENDPOINT environment variable.
+*
+* @param ctx PROJ context, or NULL
+* @param url Endpoint URL. Must NOT be NULL.
+*/
+void proj_context_set_url_endpoint(PJ_CONTEXT *ctx, const char *url) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    // Load ini file, now so as to override its network settings
+    pj_load_ini(ctx);
+    ctx->endpoint = url;
+}
+
+// ---------------------------------------------------------------------------
+
+//! @cond Doxygen_Suppress
+
+bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) {
+    if (ctx == nullptr) {
+        ctx = pj_get_default_ctx();
+    }
+    if (ctx->networking.enabled_env_variable_checked) {
+        return ctx->networking.enabled;
+    }
+    const char *enabled = getenv("PROJ_NETWORK");
+    if (enabled && enabled[0] != '\0') {
+        ctx->networking.enabled = ci_equal(enabled, "ON") ||
+                                  ci_equal(enabled, "YES") ||
+                                  ci_equal(enabled, "TRUE");
+    }
+    pj_load_ini(ctx);
+    ctx->networking.enabled_env_variable_checked = true;
+    return ctx->networking.enabled;
+}
+
+//! @endcond