From 2dfe2cdd1f4e963e6faaccd5ca29bc6d8fe4ae30 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2019 23:38:16 +0100 Subject: Add a FileManager and File class --- src/filemanager.cpp | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 src/filemanager.cpp (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp new file mode 100644 index 00000000..b6164519 --- /dev/null +++ b/src/filemanager.cpp @@ -0,0 +1,173 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "filemanager.hpp" +#include "proj_internal.h" + +NS_PROJ_START + +// --------------------------------------------------------------------------- + +File::File() = default; + +// --------------------------------------------------------------------------- + +File::~File() = default; + +// --------------------------------------------------------------------------- + +class FileStdio : public File { + PJ_CONTEXT *m_ctx; + FILE *m_fp; + + FileStdio(const FileStdio &) = delete; + FileStdio &operator=(const FileStdio &) = delete; + + protected: + FileStdio(PJ_CONTEXT *ctx, FILE *fp) : m_ctx(ctx), m_fp(fp) {} + + public: + ~FileStdio() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileStdio::~FileStdio() { fclose(m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileStdio::read(void *buffer, size_t sizeBytes) { + return fread(buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileStdio::seek(unsigned long long offset, int whence) { + // TODO one day: use 64-bit offset compatible API + if (offset != static_cast(static_cast(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return fseek(m_fp, static_cast(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileStdio::tell() { + // TODO one day: use 64-bit offset compatible API + return ftell(m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr FileStdio::open(PJ_CONTEXT *ctx, const char *filename) { + auto fp = fopen(filename, "rb"); + return std::unique_ptr(fp ? new FileStdio(ctx, fp) : nullptr); +} + +// --------------------------------------------------------------------------- + +#ifndef REMOVE_LEGACY_SUPPORT + +class FileLegacyAdapter : public File { + PJ_CONTEXT *m_ctx; + PAFile m_fp; + + FileLegacyAdapter(const FileLegacyAdapter &) = delete; + FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; + + protected: + FileLegacyAdapter(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + + public: + ~FileLegacyAdapter() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileLegacyAdapter::~FileLegacyAdapter() { pj_ctx_fclose(m_ctx, m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileLegacyAdapter::read(void *buffer, size_t sizeBytes) { + return pj_ctx_fread(m_ctx, buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileLegacyAdapter::seek(unsigned long long offset, int whence) { + if (offset != static_cast(static_cast(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return pj_ctx_fseek(m_ctx, m_fp, static_cast(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileLegacyAdapter::tell() { + return pj_ctx_ftell(m_ctx, m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, + const char *filename) { + auto fid = pj_ctx_fopen(ctx, filename, "rb"); + return std::unique_ptr(fid ? new FileLegacyAdapter(ctx, fid) + : nullptr); +} + +#endif // REMOVE_LEGACY_SUPPORT + +// --------------------------------------------------------------------------- + +std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { +#ifndef REMOVE_LEGACY_SUPPORT + // If the user has specified a legacy fileapi, use it + if (ctx->fileapi_legacy != pj_get_default_fileapi()) { + return FileLegacyAdapter::open(ctx, filename); + } +#endif + return FileStdio::open(ctx, filename); +} + +NS_PROJ_END -- cgit v1.2.3 From a06f4a258f618dbad2ce01feadab6908db00bda5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 21:24:53 +0100 Subject: Add proj_context_set_network_callbacks() with an empty implementation --- src/filemanager.cpp | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index b6164519..97f6a4dc 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -25,11 +25,21 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ +#ifndef FROM_PROJ_CPP +#define FROM_PROJ_CPP +#endif + #include "filemanager.hpp" +#include "proj.h" +#include "proj/internal/internal.hpp" #include "proj_internal.h" +//! @cond Doxygen_Suppress + NS_PROJ_START +using namespace internal; + // --------------------------------------------------------------------------- File::File() = default; @@ -160,6 +170,79 @@ std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +class NetworkFile : public File { + PJ_CONTEXT *m_ctx; + PROJ_NETWORK_HANDLE *m_handle; + unsigned long long m_pos = 0; + + NetworkFile(const NetworkFile &) = delete; + NetworkFile &operator=(const NetworkFile &) = delete; + + protected: + NetworkFile(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle) + : m_ctx(ctx), m_handle(handle) {} + + public: + ~NetworkFile() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { + std::vector buffer(16 * 1024); + size_t size_read = 0; + auto handle = ctx->networking.open(ctx, filename, buffer.size(), &buffer[0], + &size_read, ctx->networking.user_data); + return std::unique_ptr(handle ? new NetworkFile(ctx, handle) + : nullptr); +} + +// --------------------------------------------------------------------------- + +size_t NetworkFile::read(void *buffer, size_t sizeBytes) { + size_t nRead = m_ctx->networking.read_range( + m_ctx, m_handle, m_pos, sizeBytes, buffer, m_ctx->networking.user_data); + m_pos += nRead; + return nRead; +} + +// --------------------------------------------------------------------------- + +bool NetworkFile::seek(unsigned long long offset, int whence) { + if (whence == SEEK_SET) { + m_pos = offset; + } else if (whence == SEEK_CUR) { + m_pos += offset; + } else { + if (offset != 0) + return false; + const auto filesize = m_ctx->networking.get_file_size( + m_ctx, m_handle, m_ctx->networking.user_data); + if (filesize == 0) + return false; + m_pos = filesize; + } + return true; +} + +// --------------------------------------------------------------------------- + +unsigned long long NetworkFile::tell() { return m_pos; } + +// --------------------------------------------------------------------------- + +NetworkFile::~NetworkFile() { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); +} + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifndef REMOVE_LEGACY_SUPPORT // If the user has specified a legacy fileapi, use it @@ -167,7 +250,88 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { return FileLegacyAdapter::open(ctx, filename); } #endif + if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + return NetworkFile::open(ctx, filename); + } return FileStdio::open(ctx, filename); } +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE * +no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ + void * /*user_data*/) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); + return nullptr; +} + +// --------------------------------------------------------------------------- + +static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, + void * /*user_data*/) {} + +// --------------------------------------------------------------------------- + +static const char *no_op_network_get_last_error(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *, + void * /*user_data*/) { + return "Network functionality not available"; +} + +// --------------------------------------------------------------------------- + +void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { + ctx->networking.open = no_op_network_open; + ctx->networking.close = no_op_network_close; + ctx->networking.get_last_error = no_op_network_get_last_error; +} + +// --------------------------------------------------------------------------- + NS_PROJ_END + +//! @endcond + +// --------------------------------------------------------------------------- + +/** Define a custom set of callbacks for network access. + * + * All callbacks should be provided (non NULL pointers). + * + * @param ctx PROJ context, or NULL + * @param open_cbk Callback to open a remote file given its URL + * @param close_cbk Callback to close a remote file. + * @param get_header_value_cbk Callback to get HTTP headers + * @param get_file_size_cbk Callback to get the size of the remote file. + * @param read_range_cbk Callback to read a range of bytes inside a remote file. + * @param get_last_error_cbk Callback to get last error message. + * @param user_data Arbitrary pointer provided by the user, and passed to the + * above callbacks. May be NULL. + * @return TRUE in case of success. + */ +int proj_context_set_network_callbacks( + PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_get_file_size_cbk_type get_file_size_cbk, + proj_network_read_range_type read_range_cbk, + proj_network_get_last_error_type get_last_error_cbk, void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!open_cbk || !close_cbk || !get_header_value_cbk || + !get_file_size_cbk || !read_range_cbk || !get_last_error_cbk) { + return false; + } + ctx->networking.open = open_cbk; + ctx->networking.close = close_cbk; + ctx->networking.get_header_value = get_header_value_cbk; + ctx->networking.get_file_size = get_file_size_cbk; + ctx->networking.read_range = read_range_cbk; + ctx->networking.get_last_error = get_last_error_cbk; + ctx->networking.user_data = user_data; + return true; +} -- cgit v1.2.3 From f73527fa20c9250cfced6aa73a7e46f40dc2c214 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 22:36:07 +0100 Subject: Add very minimalistic and slow libcurl implementation --- src/filemanager.cpp | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 97f6a4dc..3a2acee4 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -29,11 +29,18 @@ #define FROM_PROJ_CPP #endif +#include + #include "filemanager.hpp" #include "proj.h" #include "proj/internal/internal.hpp" #include "proj_internal.h" +#ifdef CURL_ENABLED +#include +#include // for sqlite3_snprintf +#endif + //! @cond Doxygen_Suppress NS_PROJ_START @@ -258,6 +265,152 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { // --------------------------------------------------------------------------- +#ifdef CURL_ENABLED + +struct CurlFileHandle { + CURL *m_handle = nullptr; + std::string m_headers; + + CurlFileHandle(const CurlFileHandle &) = delete; + CurlFileHandle &operator=(const CurlFileHandle &) = delete; + + explicit CurlFileHandle(CURL *handle, std::string &&headers) + : m_handle(handle), m_headers(std::move(headers)) {} + ~CurlFileHandle(); +}; + +// --------------------------------------------------------------------------- + +CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } + +// --------------------------------------------------------------------------- + +static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, + void *req) { + const size_t nSize = count * nmemb; + auto pStr = static_cast(req); + if (pStr->size() + nSize > pStr->capacity()) { + // to avoid servers not honouring Range to cause excessive memory + // allocation + return 0; + } + pStr->append(static_cast(buffer), nSize); + return nmemb; +} + +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, + size_t size_to_read, void *buffer, + size_t *out_size_read, void *) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return nullptr; + curl_easy_setopt(hCurlHandle, CURLOPT_URL, url); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(hCurlHandle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(hCurlHandle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(hCurlHandle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "0-%llu", size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string headers; + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, pj_curl_write_func); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, nullptr); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + curl_easy_cleanup(hCurlHandle); + return nullptr; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + *out_size_read = std::min(size_to_read, body.size()); + + return reinterpret_cast( + new CurlFileHandle(hCurlHandle, std::move(headers))); +} + +// --------------------------------------------------------------------------- + +static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, + void * /*user_data*/) { + delete reinterpret_cast(handle); +} + +// --------------------------------------------------------------------------- + +static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, + unsigned long long offset, size_t size_to_read, + void *buffer, void *) { + auto handle = reinterpret_cast(raw_handle); + auto hCurlHandle = handle->m_handle; + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + return 0; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + return std::min(size_to_read, body.size()); +} + +#else + +// --------------------------------------------------------------------------- + static PROJ_NETWORK_HANDLE * no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, size_t, /* size to read */ @@ -281,12 +434,20 @@ static const char *no_op_network_get_last_error(PJ_CONTEXT *, return "Network functionality not available"; } +#endif + // --------------------------------------------------------------------------- void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { +#ifdef CURL_ENABLED + ctx->networking.open = pj_curl_open; + ctx->networking.close = pj_curl_close; + ctx->networking.read_range = pj_curl_read_range; +#else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; ctx->networking.get_last_error = no_op_network_get_last_error; +#endif } // --------------------------------------------------------------------------- -- cgit v1.2.3 From ed92965937856ae697010e1461e82f6245d19909 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 20 Dec 2019 19:10:20 +0100 Subject: Network: add a memory cache and I/O chunking strategy --- src/filemanager.cpp | 259 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 243 insertions(+), 16 deletions(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 3a2acee4..2dfcd6b6 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -28,14 +28,34 @@ #ifndef FROM_PROJ_CPP #define FROM_PROJ_CPP #endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include +#include +#include #include "filemanager.hpp" #include "proj.h" #include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" #include "proj_internal.h" +#ifdef __MINGW32__ +// mingw32-win32 doesn't implement std::mutex +namespace { +class MyMutex { + public: + // cppcheck-suppress functionStatic + void lock() { pj_acquire_lock(); } + // cppcheck-suppress functionStatic + void unlock() { pj_release_lock(); } +}; +} +#else +#include +#define MyMutex std::mutex +#endif + #ifdef CURL_ENABLED #include #include // for sqlite3_snprintf @@ -177,17 +197,97 @@ std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +constexpr size_t DOWNLOAD_CHUNK_SIZE = 16 * 1024; +constexpr int MAX_CHUNKS = 64; + +class NetworkChunkCache { + public: + void insert(const std::string &url, unsigned long long chunkIdx, + std::vector &&data); + + std::shared_ptr> + get(const std::string &url, unsigned long long chunkIdx); + + void clear(); + + private: + struct Key { + std::string url; + unsigned long long chunkIdx; + + Key(const std::string &urlIn, unsigned long long chunkIdxIn) + : url(urlIn), chunkIdx(chunkIdxIn) {} + bool operator==(const Key &other) const { + return url == other.url && chunkIdx == other.chunkIdx; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return std::hash{}(k.url) ^ + (std::hash{}(k.chunkIdx) << 1); + } + }; + + lru11::Cache< + Key, std::shared_ptr>, MyMutex, + std::unordered_map< + Key, + typename std::list>>>::iterator, + KeyHasher>> + cache_{MAX_CHUNKS}; +}; + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::insert(const std::string &url, + unsigned long long chunkIdx, + std::vector &&data) { + cache_.insert( + Key(url, chunkIdx), + std::make_shared>(std::move(data))); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr> +NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) { + std::shared_ptr> ret; + cache_.tryGet(Key(url, chunkIdx), ret); + return ret; +} + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::clear() +{ + cache_.clear(); +} + +// --------------------------------------------------------------------------- + +static NetworkChunkCache gNetworkChunkCache{}; + +// --------------------------------------------------------------------------- + class NetworkFile : public File { PJ_CONTEXT *m_ctx; + std::string m_url; PROJ_NETWORK_HANDLE *m_handle; unsigned long long m_pos = 0; + size_t m_nBlocksToDownload = 1; + unsigned long long m_lastDownloadedOffset; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; protected: - NetworkFile(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle) - : m_ctx(ctx), m_handle(handle) {} + NetworkFile(PJ_CONTEXT *ctx, const std::string &url, + PROJ_NETWORK_HANDLE *handle, + unsigned long long lastDownloadOffset) + : m_ctx(ctx), m_url(url), m_handle(handle), + m_lastDownloadedOffset(lastDownloadOffset) {} public: ~NetworkFile() override; @@ -202,20 +302,125 @@ class NetworkFile : public File { // --------------------------------------------------------------------------- std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { - std::vector buffer(16 * 1024); - size_t size_read = 0; - auto handle = ctx->networking.open(ctx, filename, buffer.size(), &buffer[0], - &size_read, ctx->networking.user_data); - return std::unique_ptr(handle ? new NetworkFile(ctx, handle) - : nullptr); + if (gNetworkChunkCache.get(filename, 0)) { + return std::unique_ptr( + new NetworkFile(ctx, filename, nullptr, + std::numeric_limits::max())); + } else { + std::vector buffer(DOWNLOAD_CHUNK_SIZE); + size_t size_read = 0; + auto handle = + ctx->networking.open(ctx, filename, 0, buffer.size(), &buffer[0], + &size_read, ctx->networking.user_data); + buffer.resize(size_read); + gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + return std::unique_ptr( + handle ? new NetworkFile(ctx, filename, handle, size_read) + : nullptr); + } } // --------------------------------------------------------------------------- size_t NetworkFile::read(void *buffer, size_t sizeBytes) { - size_t nRead = m_ctx->networking.read_range( - m_ctx, m_handle, m_pos, sizeBytes, buffer, m_ctx->networking.user_data); - m_pos += nRead; + + if (sizeBytes == 0) + return 0; + + auto iterOffset = m_pos; + while (sizeBytes) { + const auto chunkIdxToDownload = iterOffset / DOWNLOAD_CHUNK_SIZE; + const auto offsetToDownload = chunkIdxToDownload * DOWNLOAD_CHUNK_SIZE; + std::vector region; + auto pChunk = gNetworkChunkCache.get(m_url, chunkIdxToDownload); + if (pChunk != nullptr) { + region = *pChunk; + } else { + if (offsetToDownload == m_lastDownloadedOffset) { + // In case of consecutive reads (of small size), we use a + // heuristic that we will read the file sequentially, so + // we double the requested size to decrease the number of + // client/server roundtrips. + if (m_nBlocksToDownload < 100) + m_nBlocksToDownload *= 2; + } else { + // Random reads. Cancel the above heuristics. + m_nBlocksToDownload = 1; + } + + // Ensure that we will request at least the number of blocks + // to satisfy the remaining buffer size to read. + const auto endOffsetToDownload = + ((iterOffset + sizeBytes + DOWNLOAD_CHUNK_SIZE - 1) / + DOWNLOAD_CHUNK_SIZE) * + DOWNLOAD_CHUNK_SIZE; + const auto nMinBlocksToDownload = static_cast( + (endOffsetToDownload - offsetToDownload) / DOWNLOAD_CHUNK_SIZE); + if (m_nBlocksToDownload < nMinBlocksToDownload) + m_nBlocksToDownload = nMinBlocksToDownload; + + // Avoid reading already cached data. + // Note: this might get evicted if concurrent reads are done, but + // this should not cause bugs. Just missed optimization. + for (size_t i = 1; i < m_nBlocksToDownload; i++) { + if (gNetworkChunkCache.get(m_url, chunkIdxToDownload + i) != + nullptr) { + m_nBlocksToDownload = i; + break; + } + } + + if (m_nBlocksToDownload > MAX_CHUNKS) + m_nBlocksToDownload = MAX_CHUNKS; + + region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); + size_t nRead = 0; + if (!m_handle) { + m_handle = m_ctx->networking.open( + m_ctx, m_url.c_str(), offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + &nRead, m_ctx->networking.user_data); + if (!m_handle) { + return 0; + } + } else { + nRead = m_ctx->networking.read_range( + m_ctx, m_handle, offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + m_ctx->networking.user_data); + } + if (nRead == 0) { + return 0; + } + region.resize(nRead); + m_lastDownloadedOffset = offsetToDownload + nRead; + + const auto nChunks = + (region.size() + DOWNLOAD_CHUNK_SIZE - 1) / DOWNLOAD_CHUNK_SIZE; + for (size_t i = 0; i < nChunks; i++) { + std::vector chunk( + region.data() + i * DOWNLOAD_CHUNK_SIZE, + region.data() + + std::min((i + 1) * DOWNLOAD_CHUNK_SIZE, region.size())); + gNetworkChunkCache.insert(m_url, chunkIdxToDownload + i, + std::move(chunk)); + } + } + const size_t nToCopy = static_cast( + std::min(static_cast(sizeBytes), + region.size() - (iterOffset - offsetToDownload))); + memcpy(buffer, region.data() + iterOffset - offsetToDownload, nToCopy); + buffer = static_cast(buffer) + nToCopy; + iterOffset += nToCopy; + sizeBytes -= nToCopy; + if (region.size() < static_cast(DOWNLOAD_CHUNK_SIZE) && + sizeBytes != 0) { + break; + } + } + + size_t nRead = static_cast(iterOffset - m_pos); + m_pos = iterOffset; return nRead; } @@ -229,6 +434,16 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; + if (!m_handle) { + size_t nRead = 0; + char dummy; + m_handle = + m_ctx->networking.open(m_ctx, m_url.c_str(), 0, 1, &dummy, + &nRead, m_ctx->networking.user_data); + if (!m_handle) { + return false; + } + } const auto filesize = m_ctx->networking.get_file_size( m_ctx, m_handle, m_ctx->networking.user_data); if (filesize == 0) @@ -245,7 +460,9 @@ unsigned long long NetworkFile::tell() { return m_pos; } // --------------------------------------------------------------------------- NetworkFile::~NetworkFile() { - m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + if (m_handle) { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + } } // --------------------------------------------------------------------------- @@ -301,6 +518,7 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, + unsigned long long offset, size_t size_to_read, void *buffer, size_t *out_size_read, void *) { CURL *hCurlHandle = curl_easy_init(); @@ -326,7 +544,8 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, } char szBuffer[128]; - sqlite3_snprintf(sizeof(szBuffer), szBuffer, "0-%llu", size_to_read - 1); + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); std::string headers; @@ -413,9 +632,10 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, static PROJ_NETWORK_HANDLE * no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, - size_t, /* size to read */ - void *, /* buffer to update with bytes read*/ - size_t *, /* output: size actually read */ + unsigned long long, /* offset */ + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ void * /*user_data*/) { pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); return nullptr; @@ -452,6 +672,13 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- +void FileManager::clearCache() +{ + gNetworkChunkCache.clear(); +} + +// --------------------------------------------------------------------------- + NS_PROJ_END //! @endcond -- cgit v1.2.3 From d3bdbb841719780560438129b4911a86a7a4be58 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 21 Dec 2019 00:38:50 +0100 Subject: Add testing of network functionality --- src/filemanager.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 2dfcd6b6..cc692616 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -260,10 +260,7 @@ NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) { // --------------------------------------------------------------------------- -void NetworkChunkCache::clear() -{ - cache_.clear(); -} +void NetworkChunkCache::clear() { cache_.clear(); } // --------------------------------------------------------------------------- @@ -672,10 +669,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -void FileManager::clearCache() -{ - gNetworkChunkCache.clear(); -} +void FileManager::clearCache() { gNetworkChunkCache.clear(); } // --------------------------------------------------------------------------- -- cgit v1.2.3 From 9d0bd793b552e248a10f9ff8b6c62d942fe437a1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Dec 2019 15:58:47 +0100 Subject: Network: remove dedicated get_file_size() callback to use get_header_value(), and test --- src/filemanager.cpp | 93 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 27 deletions(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index cc692616..ea0a63ea 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -266,6 +266,13 @@ void NetworkChunkCache::clear() { cache_.clear(); } static NetworkChunkCache gNetworkChunkCache{}; +struct FileProperties { + unsigned long long size; +}; + +static lru11::Cache + gNetworkFileProperties{}; + // --------------------------------------------------------------------------- class NetworkFile : public File { @@ -275,6 +282,7 @@ class NetworkFile : public File { unsigned long long m_pos = 0; size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; + unsigned long long m_filesize; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; @@ -282,9 +290,10 @@ class NetworkFile : public File { protected: NetworkFile(PJ_CONTEXT *ctx, const std::string &url, PROJ_NETWORK_HANDLE *handle, - unsigned long long lastDownloadOffset) + unsigned long long lastDownloadOffset, + unsigned long long filesize) : m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset) {} + m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} public: ~NetworkFile() override; @@ -300,9 +309,14 @@ class NetworkFile : public File { std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { if (gNetworkChunkCache.get(filename, 0)) { - return std::unique_ptr( - new NetworkFile(ctx, filename, nullptr, - std::numeric_limits::max())); + unsigned long long filesize = 0; + FileProperties props; + if (gNetworkFileProperties.tryGet(filename, props)) { + filesize = props.size; + } + return std::unique_ptr(new NetworkFile( + ctx, filename, nullptr, + std::numeric_limits::max(), filesize)); } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; @@ -311,8 +325,24 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { &size_read, ctx->networking.user_data); buffer.resize(size_read); gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + + unsigned long long filesize = 0; + if (handle) { + const char *contentRange = ctx->networking.get_header_value( + ctx, handle, "Content-Range", ctx->networking.user_data); + if (contentRange) { + const char *slash = strchr(contentRange, '/'); + if (slash) { + filesize = std::stoull(slash + 1); + FileProperties props; + props.size = filesize; + gNetworkFileProperties.insert(filename, props); + } + } + } + return std::unique_ptr( - handle ? new NetworkFile(ctx, filename, handle, size_read) + handle ? new NetworkFile(ctx, filename, handle, size_read, filesize) : nullptr); } } @@ -431,21 +461,7 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; - if (!m_handle) { - size_t nRead = 0; - char dummy; - m_handle = - m_ctx->networking.open(m_ctx, m_url.c_str(), 0, 1, &dummy, - &nRead, m_ctx->networking.user_data); - if (!m_handle) { - return false; - } - } - const auto filesize = m_ctx->networking.get_file_size( - m_ctx, m_handle, m_ctx->networking.user_data); - if (filesize == 0) - return false; - m_pos = filesize; + m_pos = m_filesize; } return true; } @@ -484,6 +500,7 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { struct CurlFileHandle { CURL *m_handle = nullptr; std::string m_headers; + std::string m_lastval{}; CurlFileHandle(const CurlFileHandle &) = delete; CurlFileHandle &operator=(const CurlFileHandle &) = delete; @@ -623,6 +640,27 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, return std::min(size_to_read, body.size()); } +// --------------------------------------------------------------------------- + +static const char *pj_curl_get_header_value(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *raw_handle, + const char *header_name, void *) { + auto handle = reinterpret_cast(raw_handle); + auto pos = ci_find(handle->m_headers, header_name); + if (pos == std::string::npos) + return nullptr; + const char *c_str = handle->m_headers.c_str(); + if (c_str[pos] == ':') + pos++; + while (c_str[pos] == ' ') + pos++; + auto posEnd = pos; + while (c_str[posEnd] != '\n' && c_str[posEnd] != '\0') + posEnd++; + handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos); + return handle->m_lastval.c_str(); +} + #else // --------------------------------------------------------------------------- @@ -660,6 +698,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { ctx->networking.open = pj_curl_open; ctx->networking.close = pj_curl_close; ctx->networking.read_range = pj_curl_read_range; + ctx->networking.get_header_value = pj_curl_get_header_value; #else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; @@ -669,7 +708,10 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -void FileManager::clearCache() { gNetworkChunkCache.clear(); } +void FileManager::clearCache() { + gNetworkChunkCache.clear(); + gNetworkFileProperties.clear(); +} // --------------------------------------------------------------------------- @@ -687,7 +729,6 @@ NS_PROJ_END * @param open_cbk Callback to open a remote file given its URL * @param close_cbk Callback to close a remote file. * @param get_header_value_cbk Callback to get HTTP headers - * @param get_file_size_cbk Callback to get the size of the remote file. * @param read_range_cbk Callback to read a range of bytes inside a remote file. * @param get_last_error_cbk Callback to get last error message. * @param user_data Arbitrary pointer provided by the user, and passed to the @@ -698,20 +739,18 @@ int proj_context_set_network_callbacks( PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_get_file_size_cbk_type get_file_size_cbk, proj_network_read_range_type read_range_cbk, proj_network_get_last_error_type get_last_error_cbk, void *user_data) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } - if (!open_cbk || !close_cbk || !get_header_value_cbk || - !get_file_size_cbk || !read_range_cbk || !get_last_error_cbk) { + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk || + !get_last_error_cbk) { return false; } ctx->networking.open = open_cbk; ctx->networking.close = close_cbk; ctx->networking.get_header_value = get_header_value_cbk; - ctx->networking.get_file_size = get_file_size_cbk; ctx->networking.read_range = read_range_cbk; ctx->networking.get_last_error = get_last_error_cbk; ctx->networking.user_data = user_data; -- cgit v1.2.3 From 0a1f1fe469029ae31591dc8b51d20f5617496128 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Dec 2019 23:12:26 +0100 Subject: Network: only enable it if PROJ_NETWORK=ON or proj_context_set_enable_network(ctx, true) --- src/filemanager.cpp | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index ea0a63ea..cd738d5e 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -63,9 +63,9 @@ class MyMutex { //! @cond Doxygen_Suppress -NS_PROJ_START +using namespace NS_PROJ::internal; -using namespace internal; +NS_PROJ_START // --------------------------------------------------------------------------- @@ -488,6 +488,14 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { } #endif if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + if (!pj_context_is_network_enabled(ctx)) { + pj_log( + ctx, PJ_LOG_ERROR, + "Attempt at accessing remote resource not authorized. Either " + "set PROJ_NETWORK=ON or " + "proj_context_set_enable_network(ctx, TRUE)"); + return nullptr; + } return NetworkFile::open(ctx, filename); } return FileStdio::open(ctx, filename); @@ -756,3 +764,49 @@ int proj_context_set_network_callbacks( ctx->networking.user_data = user_data; return true; } + +// --------------------------------------------------------------------------- + +/** Enable or disable network access. +* +* @param ctx PROJ context, or NULL +* @param enable TRUE if network access is allowed. +* @return TRUE if network access is possible. That is either libcurl is +* available, or an alternate interface has been set. +*/ +int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + ctx->networking.enabled_env_variable_checked = true; + ctx->networking.enabled = enable != FALSE; +#ifdef CURL_ENABLED + return ctx->networking.enabled; +#else + return ctx->networking.enabled && + ctx->networking.open != NS_PROJ::no_op_network_open; +#endif +} + +// --------------------------------------------------------------------------- + +//! @cond Doxygen_Suppress + +bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (ctx->networking.enabled_env_variable_checked) { + return ctx->networking.enabled; + } + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled && enabled[0] != '\0') { + ctx->networking.enabled = ci_equal(enabled, "ON") || + ci_equal(enabled, "YES") || + ci_equal(enabled, "TRUE"); + } + ctx->networking.enabled_env_variable_checked = true; + return ctx->networking.enabled; +} + +//! @endcond -- cgit v1.2.3 From c4589fbe42e5fea07a03919d3484164f5fb70dd3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 18:44:45 +0100 Subject: Network: automatically use CDN resources when local resources not available, and networking enabled --- src/filemanager.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index cd738d5e..d9a02632 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -69,7 +69,7 @@ NS_PROJ_START // --------------------------------------------------------------------------- -File::File() = default; +File::File(const std::string &name) : name_(name) {} // --------------------------------------------------------------------------- @@ -85,7 +85,8 @@ class FileStdio : public File { FileStdio &operator=(const FileStdio &) = delete; protected: - FileStdio(PJ_CONTEXT *ctx, FILE *fp) : m_ctx(ctx), m_fp(fp) {} + FileStdio(const std::string &name, PJ_CONTEXT *ctx, FILE *fp) + : File(name), m_ctx(ctx), m_fp(fp) {} public: ~FileStdio() override; @@ -130,7 +131,8 @@ unsigned long long FileStdio::tell() { std::unique_ptr FileStdio::open(PJ_CONTEXT *ctx, const char *filename) { auto fp = fopen(filename, "rb"); - return std::unique_ptr(fp ? new FileStdio(ctx, fp) : nullptr); + return std::unique_ptr(fp ? new FileStdio(filename, ctx, fp) + : nullptr); } // --------------------------------------------------------------------------- @@ -145,7 +147,8 @@ class FileLegacyAdapter : public File { FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; protected: - FileLegacyAdapter(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + FileLegacyAdapter(const std::string &name, PJ_CONTEXT *ctx, PAFile fp) + : File(name), m_ctx(ctx), m_fp(fp) {} public: ~FileLegacyAdapter() override; @@ -189,7 +192,7 @@ unsigned long long FileLegacyAdapter::tell() { std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, const char *filename) { auto fid = pj_ctx_fopen(ctx, filename, "rb"); - return std::unique_ptr(fid ? new FileLegacyAdapter(ctx, fid) + return std::unique_ptr(fid ? new FileLegacyAdapter(filename, ctx, fid) : nullptr); } @@ -292,7 +295,7 @@ class NetworkFile : public File { PROJ_NETWORK_HANDLE *handle, unsigned long long lastDownloadOffset, unsigned long long filesize) - : m_ctx(ctx), m_url(url), m_handle(handle), + : File(url), m_ctx(ctx), m_url(url), m_handle(handle), m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} public: -- cgit v1.2.3 From 2093aca0720949303410280912b61efd791d2f01 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 20:13:03 +0100 Subject: Network: make CDN endpoint configurable either in proj.ini, PROJ_NETWORK_ENDPOINT or proj_context_set_url_endpoint() --- src/filemanager.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index d9a02632..97d7369e 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -772,6 +772,9 @@ int proj_context_set_network_callbacks( /** Enable or disable network access. * +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK environment variable. +* * @param ctx PROJ context, or NULL * @param enable TRUE if network access is allowed. * @return TRUE if network access is possible. That is either libcurl is @@ -781,6 +784,8 @@ int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); ctx->networking.enabled_env_variable_checked = true; ctx->networking.enabled = enable != FALSE; #ifdef CURL_ENABLED @@ -793,6 +798,25 @@ int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { // --------------------------------------------------------------------------- +/** Define the URL endpoint to query for remote grids. +* +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK_ENDPOINT environment variable. +* +* @param ctx PROJ context, or NULL +* @param url Endpoint URL. Must NOT be NULL. +*/ +void proj_context_set_url_endpoint(PJ_CONTEXT *ctx, const char *url) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); + ctx->endpoint = url; +} + +// --------------------------------------------------------------------------- + //! @cond Doxygen_Suppress bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { @@ -808,6 +832,7 @@ bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { ci_equal(enabled, "YES") || ci_equal(enabled, "TRUE"); } + pj_load_ini(ctx); ctx->networking.enabled_env_variable_checked = true; return ctx->networking.enabled; } -- cgit v1.2.3 From aa8c7826cf17e650ee2c3a2281aca49db37c4e81 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 22:16:28 +0100 Subject: Network: rework error handling --- src/filemanager.cpp | 95 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 25 deletions(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 97d7369e..551301c6 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -323,11 +323,19 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; - auto handle = - ctx->networking.open(ctx, filename, 0, buffer.size(), &buffer[0], - &size_read, ctx->networking.user_data); + std::string errorBuffer; + errorBuffer.resize(1024); + + auto handle = ctx->networking.open( + ctx, filename, 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); buffer.resize(size_read); gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename, + errorBuffer.c_str()); + } unsigned long long filesize = 0; if (handle) { @@ -405,11 +413,14 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); size_t nRead = 0; + std::string errorBuffer; + errorBuffer.resize(1024); if (!m_handle) { m_handle = m_ctx->networking.open( m_ctx, m_url.c_str(), offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], - &nRead, m_ctx->networking.user_data); + &nRead, errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); if (!m_handle) { return 0; } @@ -417,9 +428,15 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { nRead = m_ctx->networking.read_range( m_ctx, m_handle, offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + errorBuffer.size(), &errorBuffer[0], m_ctx->networking.user_data); } if (nRead == 0) { + errorBuffer.resize(strlen(errorBuffer.data())); + if (!errorBuffer.empty()) { + pj_log(m_ctx, PJ_LOG_ERROR, "Cannot read in %s: %s", + m_url.c_str(), errorBuffer.c_str()); + } return 0; } region.resize(nRead); @@ -542,10 +559,10 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- -static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, - unsigned long long offset, - size_t size_to_read, void *buffer, - size_t *out_size_read, void *) { +static PROJ_NETWORK_HANDLE * +pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *) { CURL *hCurlHandle = curl_easy_init(); if (!hCurlHandle) return nullptr; @@ -583,6 +600,10 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -594,10 +615,24 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } curl_easy_cleanup(hCurlHandle); return nullptr; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -619,7 +654,8 @@ static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, unsigned long long offset, size_t size_to_read, - void *buffer, void *) { + void *buffer, size_t error_string_max_size, + char *out_error_string, void *) { auto handle = reinterpret_cast(raw_handle); auto hCurlHandle = handle->m_handle; @@ -633,6 +669,10 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -641,9 +681,23 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } return 0; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -682,8 +736,12 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, size_t, /* size to read */ void *, /* buffer to update with bytes read*/ size_t *, /* output: size actually read */ + size_t error_string_max_size, char *out_error_string, void * /*user_data*/) { - pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); + if (out_error_string) { + snprintf(out_error_string, error_string_max_size, "%s", + "Network functionality not available"); + } return nullptr; } @@ -692,14 +750,6 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void * /*user_data*/) {} -// --------------------------------------------------------------------------- - -static const char *no_op_network_get_last_error(PJ_CONTEXT *, - PROJ_NETWORK_HANDLE *, - void * /*user_data*/) { - return "Network functionality not available"; -} - #endif // --------------------------------------------------------------------------- @@ -713,7 +763,6 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { #else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; - ctx->networking.get_last_error = no_op_network_get_last_error; #endif } @@ -741,7 +790,6 @@ NS_PROJ_END * @param close_cbk Callback to close a remote file. * @param get_header_value_cbk Callback to get HTTP headers * @param read_range_cbk Callback to read a range of bytes inside a remote file. - * @param get_last_error_cbk Callback to get last error message. * @param user_data Arbitrary pointer provided by the user, and passed to the * above callbacks. May be NULL. * @return TRUE in case of success. @@ -750,20 +798,17 @@ int proj_context_set_network_callbacks( PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_read_range_type read_range_cbk, - proj_network_get_last_error_type get_last_error_cbk, void *user_data) { + proj_network_read_range_type read_range_cbk, void *user_data) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } - if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk || - !get_last_error_cbk) { + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk) { return false; } ctx->networking.open = open_cbk; ctx->networking.close = close_cbk; ctx->networking.get_header_value = get_header_value_cbk; ctx->networking.read_range = read_range_cbk; - ctx->networking.get_last_error = get_last_error_cbk; ctx->networking.user_data = user_data; return true; } -- cgit v1.2.3 From f085b39752d00a296c288be42dfc69b39b73823f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 12:25:03 +0100 Subject: Handle context reassignment for Grid/GridSet/File objects --- src/filemanager.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 551301c6..dabb46e0 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -94,6 +94,7 @@ class FileStdio : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence = SEEK_SET) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -156,6 +157,7 @@ class FileLegacyAdapter : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence = SEEK_SET) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -286,6 +288,7 @@ class NetworkFile : public File { size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; unsigned long long m_filesize; + proj_network_close_cbk_type m_closeCbk; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; @@ -296,7 +299,8 @@ class NetworkFile : public File { unsigned long long lastDownloadOffset, unsigned long long filesize) : File(url), m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} + m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize), + m_closeCbk(ctx->networking.close) {} public: ~NetworkFile() override; @@ -304,6 +308,7 @@ class NetworkFile : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override; static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -500,6 +505,17 @@ NetworkFile::~NetworkFile() { // --------------------------------------------------------------------------- +void NetworkFile::reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + if (m_closeCbk != m_ctx->networking.close) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Networking close callback has changed following context " + "reassignment ! This is highly suspicious"); + } +} + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifndef REMOVE_LEGACY_SUPPORT // If the user has specified a legacy fileapi, use it -- cgit v1.2.3 From f934c002c0742dc4bb6fcda1ff44e4035f472ce8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 17:41:33 +0100 Subject: CurlFileHandle: code improvement. No functional change --- src/filemanager.cpp | 96 +++++++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 43 deletions(-) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index dabb46e0..a279a6cf 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -542,20 +542,52 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifdef CURL_ENABLED struct CurlFileHandle { - CURL *m_handle = nullptr; - std::string m_headers; + std::string m_url; + CURL *m_handle; + std::string m_headers{}; std::string m_lastval{}; + char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; CurlFileHandle(const CurlFileHandle &) = delete; CurlFileHandle &operator=(const CurlFileHandle &) = delete; - explicit CurlFileHandle(CURL *handle, std::string &&headers) - : m_handle(handle), m_headers(std::move(headers)) {} + explicit CurlFileHandle(const char *url, CURL *handle); ~CurlFileHandle(); + + static PROJ_NETWORK_HANDLE * + open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *); }; // --------------------------------------------------------------------------- +CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) + : m_url(url), m_handle(handle) { + curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(handle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); +} + +// --------------------------------------------------------------------------- + CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } // --------------------------------------------------------------------------- @@ -575,31 +607,18 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- -static PROJ_NETWORK_HANDLE * -pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, - size_t size_to_read, void *buffer, size_t *out_size_read, - size_t error_string_max_size, char *out_error_string, void *) { +PROJ_NETWORK_HANDLE *CurlFileHandle::open(PJ_CONTEXT *, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *) { CURL *hCurlHandle = curl_easy_init(); if (!hCurlHandle) return nullptr; - curl_easy_setopt(hCurlHandle, CURLOPT_URL, url); - - if (getenv("PROJ_CURL_VERBOSE")) - curl_easy_setopt(hCurlHandle, CURLOPT_VERBOSE, 1); -// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. -#if LIBCURL_VERSION_NUM >= 0x073600 - curl_easy_setopt(hCurlHandle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); -#endif - - // Enable following redirections. Requires libcurl 7.10.1 at least. - curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(hCurlHandle, CURLOPT_MAXREDIRS, 10); - - if (getenv("PROJ_UNSAFE_SSL")) { - curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L); - } + auto file = + std::unique_ptr(new CurlFileHandle(url, hCurlHandle)); char szBuffer[128]; sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, @@ -616,9 +635,7 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); - char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; - szCurlErrBuf[0] = '\0'; - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + file->m_szCurlErrBuf[0] = '\0'; curl_easy_perform(hCurlHandle); @@ -631,19 +648,16 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); - if (response_code == 0 || response_code >= 300) { if (out_error_string) { - if (szCurlErrBuf[0]) { + if (file->m_szCurlErrBuf[0]) { snprintf(out_error_string, error_string_max_size, "%s", - szCurlErrBuf); + file->m_szCurlErrBuf); } else { snprintf(out_error_string, error_string_max_size, "HTTP error %ld: %s", response_code, body.c_str()); } } - curl_easy_cleanup(hCurlHandle); return nullptr; } if (out_error_string && error_string_max_size) { @@ -655,8 +669,8 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, } *out_size_read = std::min(size_to_read, body.size()); - return reinterpret_cast( - new CurlFileHandle(hCurlHandle, std::move(headers))); + file->m_headers = std::move(headers); + return reinterpret_cast(file.release()); } // --------------------------------------------------------------------------- @@ -685,9 +699,7 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); - char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; - szCurlErrBuf[0] = '\0'; - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + handle->m_szCurlErrBuf[0] = '\0'; curl_easy_perform(hCurlHandle); @@ -697,13 +709,11 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); - if (response_code == 0 || response_code >= 300) { if (out_error_string) { - if (szCurlErrBuf[0]) { + if (handle->m_szCurlErrBuf[0]) { snprintf(out_error_string, error_string_max_size, "%s", - szCurlErrBuf); + handle->m_szCurlErrBuf); } else { snprintf(out_error_string, error_string_max_size, "HTTP error %ld: %s", response_code, body.c_str()); @@ -772,7 +782,7 @@ static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { #ifdef CURL_ENABLED - ctx->networking.open = pj_curl_open; + ctx->networking.open = CurlFileHandle::open; ctx->networking.close = pj_curl_close; ctx->networking.read_range = pj_curl_read_range; ctx->networking.get_header_value = pj_curl_get_header_value; -- cgit v1.2.3 From 28e1770f27bb335d29bfa44a5c963904007b5e73 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 17:42:04 +0100 Subject: CurlFileHandle: set UserAgent --- src/filemanager.cpp | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) (limited to 'src/filemanager.cpp') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index a279a6cf..625dca03 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -61,8 +61,22 @@ class MyMutex { #include // for sqlite3_snprintf #endif +#if defined(__linux) +#include +#elif defined(_WIN32) +#include +#elif defined(__MACH__) && defined(__APPLE__) +#include +#elif defined(__FreeBSD__) +#include +#include +#endif + //! @cond Doxygen_Suppress +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + using namespace NS_PROJ::internal; NS_PROJ_START @@ -546,6 +560,7 @@ struct CurlFileHandle { CURL *m_handle; std::string m_headers{}; std::string m_lastval{}; + std::string m_useragent{}; char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; CurlFileHandle(const CurlFileHandle &) = delete; @@ -562,6 +577,67 @@ struct CurlFileHandle { // --------------------------------------------------------------------------- +static std::string GetExecutableName() { +#if defined(__linux) + std::string path; + path.resize(1024); + const auto ret = readlink("/proc/self/exe", &path[0], path.size()); + if (ret > 0) { + path.resize(ret); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(_WIN32) + std::string path; + path.resize(1024); + if (GetModuleFileNameA(nullptr, &path[0], + static_cast(path.size()))) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('\\'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__MACH__) && defined(__APPLE__) + std::string path; + path.resize(1024); + uint32_t size = static_cast(path.size()); + if (_NSGetExecutablePath(&path[0], &size) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__FreeBSD__) + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PATHNAME; + mib[3] = -1; + std::string path; + path.resize(1024); + size_t size = path.size(); + if (sysctl(mib, 4, &path[0], &size, nullptr, 0) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#endif + + return std::string(); +} + +// --------------------------------------------------------------------------- + CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) : m_url(url), m_handle(handle) { curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); @@ -584,6 +660,16 @@ CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) } curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); + + if (getenv("PROJ_NO_USERAGENT") == nullptr) { + m_useragent = "PROJ " STR(PROJ_VERSION_MAJOR) "." STR( + PROJ_VERSION_MINOR) "." STR(PROJ_VERSION_PATCH); + const auto exeName = GetExecutableName(); + if (!exeName.empty()) { + m_useragent = exeName + " using " + m_useragent; + } + curl_easy_setopt(handle, CURLOPT_USERAGENT, m_useragent.data()); + } } // --------------------------------------------------------------------------- -- cgit v1.2.3