diff options
| -rw-r--r-- | src/filemanager.cpp | 95 | ||||
| -rw-r--r-- | src/proj.h | 18 | ||||
| -rw-r--r-- | src/proj_internal.h | 1 | ||||
| -rw-r--r-- | test/unit/test_network.cpp | 209 |
4 files changed, 252 insertions, 71 deletions
diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 97d7369e..551301c6 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -323,11 +323,19 @@ std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { } else { std::vector<unsigned char> buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; - auto handle = - ctx->networking.open(ctx, filename, 0, buffer.size(), &buffer[0], - &size_read, ctx->networking.user_data); + std::string errorBuffer; + errorBuffer.resize(1024); + + auto handle = ctx->networking.open( + ctx, filename, 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); buffer.resize(size_read); gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename, + errorBuffer.c_str()); + } unsigned long long filesize = 0; if (handle) { @@ -405,11 +413,14 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); size_t nRead = 0; + std::string errorBuffer; + errorBuffer.resize(1024); if (!m_handle) { m_handle = m_ctx->networking.open( m_ctx, m_url.c_str(), offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], - &nRead, m_ctx->networking.user_data); + &nRead, errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); if (!m_handle) { return 0; } @@ -417,9 +428,15 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { nRead = m_ctx->networking.read_range( m_ctx, m_handle, offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + errorBuffer.size(), &errorBuffer[0], m_ctx->networking.user_data); } if (nRead == 0) { + errorBuffer.resize(strlen(errorBuffer.data())); + if (!errorBuffer.empty()) { + pj_log(m_ctx, PJ_LOG_ERROR, "Cannot read in %s: %s", + m_url.c_str(), errorBuffer.c_str()); + } return 0; } region.resize(nRead); @@ -542,10 +559,10 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- -static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, - unsigned long long offset, - size_t size_to_read, void *buffer, - size_t *out_size_read, void *) { +static PROJ_NETWORK_HANDLE * +pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *) { CURL *hCurlHandle = curl_easy_init(); if (!hCurlHandle) return nullptr; @@ -583,6 +600,10 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -594,10 +615,24 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } curl_easy_cleanup(hCurlHandle); return nullptr; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -619,7 +654,8 @@ static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, unsigned long long offset, size_t size_to_read, - void *buffer, void *) { + void *buffer, size_t error_string_max_size, + char *out_error_string, void *) { auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle); auto hCurlHandle = handle->m_handle; @@ -633,6 +669,10 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -641,9 +681,23 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } return 0; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -682,8 +736,12 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, size_t, /* size to read */ void *, /* buffer to update with bytes read*/ size_t *, /* output: size actually read */ + size_t error_string_max_size, char *out_error_string, void * /*user_data*/) { - pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); + if (out_error_string) { + snprintf(out_error_string, error_string_max_size, "%s", + "Network functionality not available"); + } return nullptr; } @@ -692,14 +750,6 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void * /*user_data*/) {} -// --------------------------------------------------------------------------- - -static const char *no_op_network_get_last_error(PJ_CONTEXT *, - PROJ_NETWORK_HANDLE *, - void * /*user_data*/) { - return "Network functionality not available"; -} - #endif // --------------------------------------------------------------------------- @@ -713,7 +763,6 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { #else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; - ctx->networking.get_last_error = no_op_network_get_last_error; #endif } @@ -741,7 +790,6 @@ NS_PROJ_END * @param close_cbk Callback to close a remote file. * @param get_header_value_cbk Callback to get HTTP headers * @param read_range_cbk Callback to read a range of bytes inside a remote file. - * @param get_last_error_cbk Callback to get last error message. * @param user_data Arbitrary pointer provided by the user, and passed to the * above callbacks. May be NULL. * @return TRUE in case of success. @@ -750,20 +798,17 @@ int proj_context_set_network_callbacks( PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_read_range_type read_range_cbk, - proj_network_get_last_error_type get_last_error_cbk, void *user_data) { + proj_network_read_range_type read_range_cbk, void *user_data) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } - if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk || - !get_last_error_cbk) { + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk) { return false; } ctx->networking.open = open_cbk; ctx->networking.close = close_cbk; ctx->networking.get_header_value = get_header_value_cbk; ctx->networking.read_range = read_range_cbk; - ctx->networking.get_last_error = get_last_error_cbk; ctx->networking.user_data = user_data; return true; } @@ -369,6 +369,9 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; * headers from the server response to be able to respond to * proj_network_get_header_value_cbk_type callback. * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * * @return a non-NULL opaque handle in case of success. */ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( @@ -378,6 +381,8 @@ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( size_t size_to_read, void* buffer, size_t* out_size_read, + size_t error_string_max_size, + char* out_error_string, void* user_data); /** Network access: close callback */ @@ -396,6 +401,10 @@ typedef const char* (*proj_network_get_header_value_cbk_type)( * * Read size_to_read bytes from handle, starting at offset, into * buffer. + * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * * @return the number of bytes actually read (0 in case of error) */ typedef size_t (*proj_network_read_range_type)( @@ -404,12 +413,8 @@ typedef size_t (*proj_network_read_range_type)( unsigned long long offset, size_t size_to_read, void* buffer, - void* user_data); - -/** Network access: get last error message */ -typedef const char* (*proj_network_get_last_error_type)( - PJ_CONTEXT* ctx, - PROJ_NETWORK_HANDLE*, + size_t error_string_max_size, + char* out_error_string, void* user_data); int PROJ_DLL proj_context_set_network_callbacks( @@ -418,7 +423,6 @@ int PROJ_DLL proj_context_set_network_callbacks( proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, proj_network_read_range_type read_range_cbk, - proj_network_get_last_error_type get_last_error_cbk, void* user_data); int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, diff --git a/src/proj_internal.h b/src/proj_internal.h index 669fd2b5..27b30954 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -674,7 +674,6 @@ struct projNetworkCallbacksAndData proj_network_close_cbk_type close = nullptr; proj_network_get_header_value_cbk_type get_header_value = nullptr; proj_network_read_range_type read_range = nullptr; - proj_network_get_last_error_type get_last_error = nullptr; void* user_data = nullptr; }; diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index 3e64329e..89d66797 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -174,6 +174,7 @@ struct OpenEvent : public Event { unsigned long long offset = 0; size_t size_to_read = 0; std::vector<unsigned char> response{}; + std::string errorMsg{}; int file_id = 0; }; @@ -197,13 +198,10 @@ struct ReadRangeEvent : public Event { unsigned long long offset = 0; size_t size_to_read = 0; std::vector<unsigned char> response{}; + std::string errorMsg{}; int file_id = 0; }; -struct GetLastErrorEvent : public Event { - GetLastErrorEvent() { type = "GetLastErrorEvent"; } -}; - struct File {}; struct ExchangeWithCallback { @@ -220,7 +218,9 @@ struct ExchangeWithCallback { static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url, unsigned long long offset, size_t size_to_read, void *buffer, - size_t *out_size_read, void *user_data) { + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *user_data) { auto exchange = static_cast<ExchangeWithCallback *>(user_data); if (exchange->error) return nullptr; @@ -251,9 +251,14 @@ static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url, exchange->error = true; return nullptr; } + if (!openEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + openEvent->errorMsg.c_str()); + return nullptr; + } + memcpy(buffer, openEvent->response.data(), openEvent->response.size()); *out_size_read = openEvent->response.size(); - auto handle = reinterpret_cast<PROJ_NETWORK_HANDLE *>(new File()); exchange->mapIdToHandle[openEvent->file_id] = handle; return handle; @@ -336,7 +341,8 @@ static const char *get_header_value_cbk(PJ_CONTEXT *ctx, static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, unsigned long long offset, size_t size_to_read, - void *buffer, void *user_data) { + void *buffer, size_t error_string_max_size, + char *out_error_string, void *user_data) { auto exchange = static_cast<ExchangeWithCallback *>(user_data); if (exchange->error) return 0; @@ -369,43 +375,24 @@ static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, exchange->error = true; return 0; } + exchange->nextEvent++; + if (!readRangeEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + readRangeEvent->errorMsg.c_str()); + return 0; + } memcpy(buffer, readRangeEvent->response.data(), readRangeEvent->response.size()); - exchange->nextEvent++; return readRangeEvent->response.size(); } -static const char *get_last_error_cbk(PJ_CONTEXT * /*ctx*/, - PROJ_NETWORK_HANDLE *, void *user_data) { - auto exchange = static_cast<ExchangeWithCallback *>(user_data); - if (exchange->error) - return ""; - if (exchange->nextEvent >= exchange->events.size()) { - fprintf(stderr, "unexpected call to get_last_error()\n"); - exchange->error = true; - return ""; - } - auto getLastErrorEvent = dynamic_cast<GetLastErrorEvent *>( - exchange->events[exchange->nextEvent].get()); - if (!getLastErrorEvent) { - fprintf( - stderr, - "unexpected call to get_last_error(). Was expecting a %s event\n", - exchange->events[exchange->nextEvent]->type.c_str()); - exchange->error = true; - return ""; - } - exchange->nextEvent++; - return ""; -} - TEST(networking, custom) { auto ctx = proj_context_create(); proj_context_set_enable_network(ctx, true); ExchangeWithCallback exchange; - ASSERT_TRUE(proj_context_set_network_callbacks( - ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, - get_last_error_cbk, &exchange)); + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); { std::unique_ptr<OpenEvent> event(new OpenEvent()); @@ -539,9 +526,9 @@ TEST(networking, getfilesize) { auto ctx = proj_context_create(); proj_context_set_enable_network(ctx, true); ExchangeWithCallback exchange; - ASSERT_TRUE(proj_context_set_network_callbacks( - ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, - get_last_error_cbk, &exchange)); + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); { std::unique_ptr<OpenEvent> event(new OpenEvent()); @@ -600,6 +587,152 @@ TEST(networking, getfilesize) { // --------------------------------------------------------------------------- +TEST(networking, simul_open_error) { + auto ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/open_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->errorMsg = "Cannot open file"; + event->file_id = 1; + + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/open_error.tif +multiplier=1"); + + ASSERT_EQ(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, simul_read_range_error) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "dummy"; // dummy value: not used + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create(ctx, "+proj=vgridshift " + "+grids=https://foo/read_range_error.tif " + "+multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<ReadRangeEvent> event(new ReadRangeEvent()); + event->ctx = ctx; + event->offset = 3670016; + event->size_to_read = 278528; + event->errorMsg = "read range error"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = -49 / 180. * M_PI; + double z = 0; + proj_log_func(ctx, nullptr, silent_logger); + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, HUGE_VAL); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + #ifdef CURL_ENABLED TEST(networking, curl_hgridshift) { |