From 1b702a82f5e009745f5ca8bbed92579e33f598d4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 6 Dec 2019 20:23:54 +0100 Subject: Build and CI: add libtiff dependency --- src/Makefile.am | 4 ++-- src/lib_proj.cmake | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index a12de4e1..f58e57fd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ + -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ @@ -43,7 +43,7 @@ geodtest_LDADD = libproj.la lib_LTLIBRARIES = libproj.la libproj_la_LDFLAGS = -no-undefined -version-info 17:0:2 -libproj_la_LIBADD = @SQLITE3_LIBS@ +libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ libproj_la_SOURCES = \ pj_list.h proj_internal.h \ diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index f6112aef..f1547afe 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -435,8 +435,8 @@ if(USE_THREAD AND Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) target_link_libraries(${PROJ_CORE_TARGET} ${CMAKE_THREAD_LIBS_INIT}) endif() -include_directories(${SQLITE3_INCLUDE_DIR}) -target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY}) +include_directories(${SQLITE3_INCLUDE_DIR} ${TIFF_INCLUDE_DIR}) +target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY} ${TIFF_LIBRARY}) if(MSVC) target_compile_definitions(${PROJ_CORE_TARGET} -- cgit v1.2.3 From 9f908ae47cfa70d3cdb2709a8ab5d8eeb10034fc Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Dec 2019 23:26:39 +0100 Subject: Horizontal shift grids: hide the 'positive longitude shift value = westward correction' in the CTable2/NTv1/NTv2 backends --- src/apply_gridshift.cpp | 8 ++++---- src/grids.cpp | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp index a0ffa394..c786a50a 100644 --- a/src/apply_gridshift.cpp +++ b/src/apply_gridshift.cpp @@ -212,12 +212,12 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { return t; if (!inverse) { - in.lam -= t.lam; + in.lam += t.lam; in.phi += t.phi; return in; } - t.lam = tb.lam + t.lam; + t.lam = tb.lam - t.lam; t.phi = tb.phi - t.phi; do { @@ -235,8 +235,8 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { if (del.lam == HUGE_VAL) break; - dif.lam = t.lam - del.lam - tb.lam; - dif.phi = t.phi + del.phi - tb.phi; + dif.lam = t.lam + del.lam - tb.lam; + dif.phi = t.phi + del.phi - tb.phi; t.lam -= dif.lam; t.phi -= dif.phi; diff --git a/src/grids.cpp b/src/grids.cpp index 7d19b1f7..cc954542 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -469,7 +469,8 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { } /* convert seconds to radians */ latShift = static_cast(two_doubles[0] * ((M_PI / 180.0) / 3600.0)); - lonShift = static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = -static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); return true; } @@ -572,7 +573,8 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, } latShift = two_floats[1]; - lonShift = two_floats[0]; + // west longitude positive convention ! + lonShift = -two_floats[0]; return true; } @@ -642,7 +644,8 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { } /* convert seconds to radians */ latShift = static_cast(two_float[0] * ((M_PI / 180.0) / 3600.0)); - lonShift = static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = -static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); return true; } -- cgit v1.2.3 From 1f01ecb90a751c8099a50e066616639f63522134 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 6 Dec 2019 21:08:59 +0100 Subject: Add support for horizontal and vertical grids in GeoTIFF --- src/grids.cpp | 1230 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/grids.hpp | 3 + 2 files changed, 1233 insertions(+) (limited to 'src') diff --git a/src/grids.cpp b/src/grids.cpp index cc954542..010ddb22 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -34,6 +34,11 @@ #include "proj/internal/internal.hpp" #include "proj_internal.h" +#include "tiffio.h" + +#include +#include + NS_PROJ_START using namespace internal; @@ -64,12 +69,28 @@ static void swap_words(void *dataIn, size_t word_size, size_t word_count) } } +// --------------------------------------------------------------------------- + bool ExtentAndRes::fullWorldLongitude() const { return eastLon - westLon + resLon >= 2 * M_PI - 1e-10; } // --------------------------------------------------------------------------- +bool ExtentAndRes::contains(const ExtentAndRes &other) const { + return other.westLon >= westLon && other.eastLon <= eastLon && + other.southLat >= southLat && other.northLat <= northLat; +} + +// --------------------------------------------------------------------------- + +bool ExtentAndRes::intersects(const ExtentAndRes &other) const { + return other.westLon < eastLon && westLon <= other.westLon && + other.southLat < northLat && southLat <= other.northLat; +} + +// --------------------------------------------------------------------------- + Grid::Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn) : m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} @@ -246,6 +267,926 @@ VerticalShiftGridSet::~VerticalShiftGridSet() = default; // --------------------------------------------------------------------------- +static bool IsTIFF(size_t header_size, const unsigned char *header) { + // Test combinations of signature for ClassicTIFF/BigTIFF little/big endian + return header_size >= 4 && (((header[0] == 'I' && header[1] == 'I') || + (header[0] == 'M' && header[1] == 'M')) && + ((header[2] == 0x2A && header[3] == 0) || + (header[3] == 0x2A && header[2] == 0) || + (header[2] == 0x2B && header[3] == 0) || + (header[3] == 0x2B && header[2] == 0))); +} + +// --------------------------------------------------------------------------- + +enum class TIFFDataType { Int16, UInt16, Int32, UInt32, Float32, Float64 }; + +// --------------------------------------------------------------------------- + +constexpr uint16 TIFFTAG_GEOPIXELSCALE = 33550; +constexpr uint16 TIFFTAG_GEOTIEPOINTS = 33922; +constexpr uint16 TIFFTAG_GEOTRANSMATRIX = 34264; +constexpr uint16 TIFFTAG_GEOKEYDIRECTORY = 34735; +constexpr uint16 TIFFTAG_GEODOUBLEPARAMS = 34736; +constexpr uint16 TIFFTAG_GEOASCIIPARAMS = 34737; +constexpr uint16 TIFFTAG_GDAL_METADATA = 42112; +constexpr uint16 TIFFTAG_GDAL_NODATA = 42113; + +// --------------------------------------------------------------------------- + +class GTiffGrid : public Grid { + PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset + TIFF *m_hTIFF; // owned by the belonging GTiffDataset + TIFFDataType m_dt; + uint16 m_samplesPerPixel; + uint16 m_planarConfig; + bool m_bottomUp; + toff_t m_dirOffset; + bool m_tiled; + uint32 m_blockWidth = 0; + uint32 m_blockHeight = 0; + mutable int m_lastSample = -1; + mutable int m_lastBlockX = -1; + mutable int m_lastBlockY = -1; + mutable std::vector m_buffer{}; + unsigned m_blocksPerRow = 0; + unsigned m_blocksPerCol = 0; + std::map m_mapOffset{}; + std::map m_mapScale{}; + std::map, std::string> m_metadata{}; + bool m_hasNodata = false; + float m_noData = 0.0f; + uint32 m_subfileType = 0; + + GTiffGrid(const GTiffGrid &) = delete; + GTiffGrid &operator=(const GTiffGrid &) = delete; + + void getScaleOffset(double &scale, double &offset, uint16 sample) const; + + template + float readValue(uint32 offsetInBlock, uint16 sample) const; + + public: + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, + const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); + + ~GTiffGrid() override; + + uint16 samplesPerPixel() const { return m_samplesPerPixel; } + + bool valueAt(uint16 sample, int x, int y, float &out) const; + + bool isNodata(float val) const; + + std::string metadataItem(const std::string &key, int sample = -1) const; + + uint32 subfileType() const { return m_subfileType; } +}; + +// --------------------------------------------------------------------------- + +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, + const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, + bool bottomUpIn) + : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), m_dt(dtIn), + m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), + m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), + m_tiled(TIFFIsTiled(hTIFF) != 0) { + + if (m_tiled) { + TIFFGetField(m_hTIFF, TIFFTAG_TILEWIDTH, &m_blockWidth); + TIFFGetField(m_hTIFF, TIFFTAG_TILELENGTH, &m_blockHeight); + } else { + m_blockWidth = widthIn; + TIFFGetField(m_hTIFF, TIFFTAG_ROWSPERSTRIP, &m_blockHeight); + if (m_blockHeight > static_cast(m_height)) + m_blockHeight = m_height; + } + + TIFFGetField(m_hTIFF, TIFFTAG_SUBFILETYPE, &m_subfileType); + + m_blocksPerRow = (m_width + m_blockWidth - 1) / m_blockWidth; + m_blocksPerCol = (m_height + m_blockHeight - 1) / m_blockHeight; + + const char *text = nullptr; + // Poor-man XML parsing of TIFFTAG_GDAL_METADATA tag. Hopefully good + // enough for our purposes. + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_METADATA, &text)) { + const char *ptr = text; + while (true) { + ptr = strstr(ptr, "'); + if (endTag == nullptr) + break; + const char *endValue = strchr(endTag, '<'); + if (endValue == nullptr) + break; + + std::string tag; + tag.append(ptr, endTag - ptr); + + std::string value; + value.append(endTag + 1, endValue - (endTag + 1)); + + std::string name; + auto namePos = tag.find("name=\""); + if (namePos == std::string::npos) + break; + { + namePos += strlen("name=\""); + const auto endQuote = tag.find('"', namePos); + if (endQuote == std::string::npos) + break; + name = tag.substr(namePos, endQuote - namePos); + } + + const auto samplePos = tag.find("sample=\""); + int sample = -1; + if (samplePos != std::string::npos) { + sample = atoi(tag.c_str() + samplePos + strlen("sample=\"")); + } + + m_metadata[std::pair(sample, name)] = value; + + auto rolePos = tag.find("role=\""); + if (rolePos != std::string::npos) { + rolePos += strlen("role=\""); + const auto endQuote = tag.find('"', rolePos); + if (endQuote == std::string::npos) + break; + const auto role = tag.substr(rolePos, endQuote - rolePos); + if (role == "offset") { + if (sample >= 0) { + try { + m_mapOffset[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } else if (role == "scale") { + if (sample >= 0) { + try { + m_mapScale[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } + } + + ptr = endValue + 1; + } + } + + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_NODATA, &text)) { + try { + m_noData = static_cast(c_locale_stod(text)); + m_hasNodata = true; + } catch (const std::exception &) { + } + } +} + +// --------------------------------------------------------------------------- + +GTiffGrid::~GTiffGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffGrid::getScaleOffset(double &scale, double &offset, + uint16 sample) const { + { + auto iter = m_mapScale.find(sample); + if (iter != m_mapScale.end()) + scale = iter->second; + } + + { + auto iter = m_mapOffset.find(sample); + if (iter != m_mapOffset.end()) + offset = iter->second; + } +} + +// --------------------------------------------------------------------------- + +template +float GTiffGrid::readValue(uint32 offsetInBlock, uint16 sample) const { + const auto ptr = reinterpret_cast(m_buffer.data()); + assert(offsetInBlock < m_buffer.size() / sizeof(T)); + const auto val = ptr[offsetInBlock]; + if (!m_hasNodata || static_cast(val) != m_noData) { + double scale = 1; + double offset = 0; + getScaleOffset(scale, offset, sample); + return static_cast(val * scale + offset); + } else { + return static_cast(val); + } +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, + float &out) const { + assert(x >= 0 && yFromBottom >= 0 && x < m_width && yFromBottom < m_height); + assert(sample < m_samplesPerPixel); + if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && + !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { + return false; + } + if (m_buffer.empty()) { + const auto blockSize = static_cast( + m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); + try { + m_buffer.resize(blockSize); + } catch (const std::exception &e) { + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + return false; + } + } + + const int blockX = x / m_blockWidth; + + // All non-TIFF grids have the first rows in the file being the one + // corresponding to the southern-most row. In GeoTIFF, the convention is + // *generally* different (when m_bottomUp == false), TIFF being an + // image-oriented image. If m_bottomUp == true, then we had GeoTIFF hints + // that the first row of the image is the southern-most. + const int yTIFF = m_bottomUp ? yFromBottom : m_height - 1 - yFromBottom; + const int blockY = yTIFF / m_blockHeight; + + if (m_lastSample != static_cast(sample) || m_lastBlockX != blockX || + m_lastBlockY != blockY) { + uint32 blockId = blockY * m_blocksPerRow + blockX; + if (m_planarConfig == PLANARCONFIG_SEPARATE) { + blockId += sample * m_blocksPerCol * m_blocksPerRow; + } + if (m_tiled) { + if (TIFFReadEncodedTile(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } else { + if (TIFFReadEncodedStrip(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } + m_lastSample = sample; + m_lastBlockX = blockX; + m_lastBlockY = blockY; + } + uint32 offsetInBlock = + (x % m_blockWidth) + (yTIFF % m_blockHeight) * m_blockWidth; + if (m_planarConfig == PLANARCONFIG_CONTIG) + offsetInBlock = offsetInBlock * m_samplesPerPixel + sample; + + switch (m_dt) { + case TIFFDataType::Int16: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::UInt16: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::Int32: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::UInt32: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::Float32: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::Float64: + out = readValue(offsetInBlock, sample); + break; + } + + return true; +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::isNodata(float val) const { + return (m_hasNodata && val == m_noData) || std::isnan(val); +} + +// --------------------------------------------------------------------------- + +std::string GTiffGrid::metadataItem(const std::string &key, int sample) const { + auto iter = m_metadata.find(std::pair(sample, key)); + if (iter == m_metadata.end()) { + return std::string(); + } + return iter->second; +} + +// --------------------------------------------------------------------------- + +class GTiffDataset { + PJ_CONTEXT *m_ctx; + PAFile m_fp; + TIFF *m_hTIFF = nullptr; + bool m_hasNextGrid = false; + toff_t m_nextDirOffset = 0; + std::string m_filename{}; + + GTiffDataset(const GTiffDataset &) = delete; + GTiffDataset &operator=(const GTiffDataset &) = delete; + + // libtiff I/O routines + static tsize_t tiffReadProc(thandle_t fd, tdata_t buf, tsize_t size) { + GTiffDataset *self = static_cast(fd); + return pj_ctx_fread(self->m_ctx, buf, 1, size, self->m_fp); + } + + static tsize_t tiffWriteProc(thandle_t, tdata_t, tsize_t) { + assert(false); + return 0; + } + + static toff_t tiffSeekProc(thandle_t fd, toff_t off, int whence) { + GTiffDataset *self = static_cast(fd); + // FIXME Remove cast to long when pj_ctx_fseek supports unsigned long + // long + if (pj_ctx_fseek(self->m_ctx, self->m_fp, static_cast(off), + whence) == 0) + return static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); + else + return static_cast(-1); + } + + static int tiffCloseProc(thandle_t) { + // done in destructor + return 0; + } + + static toff_t tiffSizeProc(thandle_t fd) { + GTiffDataset *self = static_cast(fd); + const auto old_off = pj_ctx_ftell(self->m_ctx, self->m_fp); + pj_ctx_fseek(self->m_ctx, self->m_fp, 0, SEEK_END); + const auto file_size = + static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); + pj_ctx_fseek(self->m_ctx, self->m_fp, old_off, SEEK_SET); + return file_size; + } + + static int tiffMapProc(thandle_t, tdata_t *, toff_t *) { return (0); } + + static void tiffUnmapProc(thandle_t, tdata_t, toff_t) {} + + public: + GTiffDataset(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + virtual ~GTiffDataset(); + + bool openTIFF(const std::string &filename); + + std::unique_ptr nextGrid(); +}; + +// --------------------------------------------------------------------------- + +GTiffDataset::~GTiffDataset() { + if (m_hTIFF) + TIFFClose(m_hTIFF); + pj_ctx_fclose(m_ctx, m_fp); +} + +// --------------------------------------------------------------------------- +class OneTimeTIFFTagInit { + + static TIFFExtendProc ParentExtender; + + // Function called by libtiff when initializing a TIFF directory + static void GTiffTagExtender(TIFF *tif) { + static const TIFFFieldInfo xtiffFieldInfo[] = { + // GeoTIFF tags + {TIFFTAG_GEOPIXELSCALE, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoPixelScale")}, + {TIFFTAG_GEOTIEPOINTS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoTiePoints")}, + {TIFFTAG_GEOTRANSMATRIX, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoTransformationMatrix")}, + + {TIFFTAG_GEOKEYDIRECTORY, -1, -1, TIFF_SHORT, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoKeyDirectory")}, + {TIFFTAG_GEODOUBLEPARAMS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoDoubleParams")}, + {TIFFTAG_GEOASCIIPARAMS, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast("GeoASCIIParams")}, + + // GDAL tags + {TIFFTAG_GDAL_METADATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast("GDALMetadata")}, + {TIFFTAG_GDAL_NODATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, FALSE, + const_cast("GDALNoDataValue")}, + + }; + + if (ParentExtender) + (*ParentExtender)(tif); + + TIFFMergeFieldInfo(tif, xtiffFieldInfo, + sizeof(xtiffFieldInfo) / sizeof(xtiffFieldInfo[0])); + } + + public: + OneTimeTIFFTagInit() { + assert(ParentExtender == nullptr); + // Install our TIFF tag extender + ParentExtender = TIFFSetTagExtender(GTiffTagExtender); + } +}; + +TIFFExtendProc OneTimeTIFFTagInit::ParentExtender = nullptr; + +// --------------------------------------------------------------------------- + +bool GTiffDataset::openTIFF(const std::string &filename) { + static OneTimeTIFFTagInit oneTimeTIFFTagInit; + m_hTIFF = + TIFFClientOpen(filename.c_str(), "r", static_cast(this), + GTiffDataset::tiffReadProc, GTiffDataset::tiffWriteProc, + GTiffDataset::tiffSeekProc, GTiffDataset::tiffCloseProc, + GTiffDataset::tiffSizeProc, GTiffDataset::tiffMapProc, + GTiffDataset::tiffUnmapProc); + + m_filename = filename; + m_hasNextGrid = true; + return m_hTIFF != nullptr; +} +// --------------------------------------------------------------------------- + +std::unique_ptr GTiffDataset::nextGrid() { + if (!m_hasNextGrid) + return nullptr; + if (m_nextDirOffset) { + TIFFSetSubDirectory(m_hTIFF, m_nextDirOffset); + } + + uint32 width = 0; + uint32 height = 0; + TIFFGetField(m_hTIFF, TIFFTAG_IMAGEWIDTH, &width); + TIFFGetField(m_hTIFF, TIFFTAG_IMAGELENGTH, &height); + if (width == 0 || height == 0 || width > INT_MAX || height > INT_MAX) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid image size"); + return nullptr; + } + + uint16 samplesPerPixel = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLESPERPIXEL, &samplesPerPixel)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SamplesPerPixel tag"); + return nullptr; + } + if (samplesPerPixel == 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid SamplesPerPixel value"); + return nullptr; + } + + uint16 bitsPerSample = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_BITSPERSAMPLE, &bitsPerSample)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing BitsPerSample tag"); + return nullptr; + } + + uint16 planarConfig = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PLANARCONFIG, &planarConfig)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing PlanarConfig tag"); + return nullptr; + } + + uint16 sampleFormat = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLEFORMAT, &sampleFormat)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SampleFormat tag"); + return nullptr; + } + + TIFFDataType dt; + if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 16) + dt = TIFFDataType::Int16; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 16) + dt = TIFFDataType::UInt16; + else if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 32) + dt = TIFFDataType::Int32; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 32) + dt = TIFFDataType::UInt32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 32) + dt = TIFFDataType::Float32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 64) + dt = TIFFDataType::Float64; + else { + pj_log( + m_ctx, PJ_LOG_ERROR, + "Unsupported combination of SampleFormat and BitsPerSample values"); + return nullptr; + } + + uint16 photometric = PHOTOMETRIC_MINISBLACK; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PHOTOMETRIC, &photometric)) + photometric = PHOTOMETRIC_MINISBLACK; + if (photometric != PHOTOMETRIC_MINISBLACK) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported Photometric value"); + return nullptr; + } + + uint16 compression = COMPRESSION_NONE; + if (!TIFFGetField(m_hTIFF, TIFFTAG_COMPRESSION, &compression)) + compression = COMPRESSION_NONE; + + if (compression != COMPRESSION_NONE && + !TIFFIsCODECConfigured(compression)) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Cannot open TIFF file due to missing codec."); + return nullptr; + } + // We really don't want to try dealing with old-JPEG images + if (compression == COMPRESSION_OJPEG) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported compression method."); + return nullptr; + } + + const auto blockSize = TIFFIsTiled(m_hTIFF) ? TIFFTileSize64(m_hTIFF) + : TIFFStripSize64(m_hTIFF); + if (blockSize == 0 || blockSize > 64 * 1024 * 2014) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported block size."); + return nullptr; + } + + unsigned short count = 0; + unsigned short *geokeys = nullptr; + bool pixelIsArea = false; + if (!TIFFGetField(m_hTIFF, TIFFTAG_GEOKEYDIRECTORY, &count, &geokeys)) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, "No GeoKeys tag"); + } else { + if (count < 4 || (count % 4) != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoKeys tag"); + return nullptr; + } + + if (geokeys[0] != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported GeoTIFF major version"); + return nullptr; + } + // We only know that we support GeoTIFF 1.0 and 1.1 at that time + if (geokeys[1] != 1 || geokeys[2] > 1) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, + "GeoTIFF %d.%d possibly not handled", geokeys[1], + geokeys[2]); + } + + for (unsigned int i = 4; i + 3 < count; i += 4) { + constexpr unsigned short GTModelTypeGeoKey = 1024; + constexpr unsigned short ModelTypeGeographic = 2; + + constexpr unsigned short GTRasterTypeGeoKey = 1025; + constexpr unsigned short RasterPixelIsArea = 1; + // constexpr unsigned short RasterPixelIsPoint = 2; + + if (geokeys[i] == GTModelTypeGeoKey) { + if (geokeys[i + 3] != ModelTypeGeographic) { + pj_log(m_ctx, PJ_LOG_ERROR, "Only GTModelTypeGeoKey = " + "ModelTypeGeographic is " + "supported"); + return nullptr; + } + } else if (geokeys[i] == GTRasterTypeGeoKey) { + if (geokeys[i + 3] == RasterPixelIsArea) { + pixelIsArea = true; + } + } + } + } + + double hRes = 0; + double vRes = 0; + double westLon = 0; + double northLat = 0; + + double *matrix = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTRANSMATRIX, &count, &matrix) && + count == 16) { + // If using GDAL to produce a bottom-up georeferencing, it will produce + // a GeoTransformationMatrix, since negative values in GeoPixelScale + // have historically been implementation bugs. + if (matrix[1] != 0 || matrix[4] != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Rotational terms not supported in " + "GeoTransformationMatrix tag"); + return nullptr; + } + + westLon = matrix[3]; + hRes = matrix[0]; + northLat = matrix[7]; + vRes = -matrix[5]; // negation to simulate GeoPixelScale convention + } else { + double *geopixelscale = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOPIXELSCALE, &count, + &geopixelscale) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoPixelScale tag"); + return nullptr; + } + if (count != 3) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoPixelScale tag"); + return nullptr; + } + hRes = geopixelscale[0]; + vRes = geopixelscale[1]; + + double *geotiepoints = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTIEPOINTS, &count, + &geotiepoints) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoTiePoints tag"); + return nullptr; + } + if (count != 6) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoTiePoints tag"); + return nullptr; + } + + westLon = geotiepoints[3] - geotiepoints[0] * hRes; + northLat = geotiepoints[4] + geotiepoints[1] * vRes; + } + + if (pixelIsArea) { + westLon += 0.5 * hRes; + northLat -= 0.5 * vRes; + } + + ExtentAndRes extent; + extent.westLon = westLon * DEG_TO_RAD; + extent.northLat = northLat * DEG_TO_RAD; + extent.resLon = hRes * DEG_TO_RAD; + extent.resLat = fabs(vRes) * DEG_TO_RAD; + extent.eastLon = (westLon + hRes * (width - 1)) * DEG_TO_RAD; + extent.southLat = (northLat - vRes * (height - 1)) * DEG_TO_RAD; + + if (vRes < 0) { + std::swap(extent.northLat, extent.southLat); + } + + if (!(fabs(extent.westLon) <= 4 * M_PI && + fabs(extent.eastLon) <= 4 * M_PI && + fabs(extent.northLat) <= M_PI + 1e-5 && + fabs(extent.southLat) <= M_PI + 1e-5 && + extent.westLon < extent.eastLon && + extent.southLat < extent.northLat && extent.resLon > 1e-10 && + extent.resLat > 1e-10)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Inconsistent georeferencing for %s", + m_filename.c_str()); + return nullptr; + } + + auto ret = std::unique_ptr( + new GTiffGrid(m_ctx, m_hTIFF, width, height, extent, dt, + samplesPerPixel, planarConfig, vRes < 0)); + m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; + m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); + return ret; +} + +// --------------------------------------------------------------------------- + +class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { + + GTiffVGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + + public: + ~GTiffVGridShiftSet() override; + + static std::unique_ptr + open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); +}; + +// --------------------------------------------------------------------------- + +template +static void +insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids) { + const auto &extent = grid->extentAndRes(); + + // If we have one or both of grid_name and parent_name, try to use + // the names to recreate the hiearchy + if (!gridName.empty()) { + if (mapGrids.find(gridName) != mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Several grids called %s found!", + gridName.c_str()); + } + mapGrids[gridName] = grid.get(); + } + bool gridInserted = false; + if (!parentName.empty()) { + auto iter = mapGrids.find(parentName); + if (iter == mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to non-existing parent %s. " + "Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } else { + if (iter->second->extentAndRes().contains(extent)) { + iter->second->m_children.emplace_back(std::move(grid)); + gridInserted = true; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to parent %s, but its extent is " + "not included in it. Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } + } + } else if (!gridName.empty()) { + topGrids.emplace_back(std::move(grid)); + gridInserted = true; + } + + // Fallback to analyzing spatial extents + if (!gridInserted) { + for (const auto &candidateParent : topGrids) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(grid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + topGrids.emplace_back(std::move(grid)); + } + } +} + +// --------------------------------------------------------------------------- + +class GTiffVGrid : public VerticalShiftGrid { + friend void insertIntoHierarchy( + PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids); + + std::unique_ptr m_grid; + uint16 m_idxSample; + + public: + GTiffVGrid(std::unique_ptr &&grid, uint16 idxSample); + + ~GTiffVGrid() override; + + bool valueAt(int x, int y, float &out) const override { + return m_grid->valueAt(m_idxSample, x, y, out); + } + + bool isNodata(float val, double /* multiplier */) const override { + return m_grid->isNodata(val); + } + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); +}; + +// --------------------------------------------------------------------------- + +GTiffVGridShiftSet::~GTiffVGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffVGrid::GTiffVGrid(std::unique_ptr &&grid, uint16 idxSample) + : VerticalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + m_grid(std::move(grid)), m_idxSample(idxSample) {} + +// --------------------------------------------------------------------------- + +GTiffVGrid::~GTiffVGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffVGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr +GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &filename) { + auto set = + std::unique_ptr(new GTiffVGridShiftSet(ctx, fp)); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + uint16 idxSample = 0; + + std::map mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + // Identify the index of the geoid_undulation/vertical_offset + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForShift = false; + for (int i = 0; i < static_cast(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "geoid_undulation" || desc == "vertical_offset") { + idxSample = static_cast(i); + foundDescriptionForShift = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForShift) { + if (ifd > 0) { + // Assuming that extra IFD without our channel of interest + // can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "geoid_undulation/vertical_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "geoid_undulation/vertical_offset channel"); + return nullptr; + } + } + } + + if (idxSample >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_name"); + + auto vgrid = + internal::make_unique(std::move(grid), idxSample); + + insertIntoHierarchy(ctx, std::move(vgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} + +// --------------------------------------------------------------------------- + std::unique_ptr VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (filename == "null") { @@ -277,6 +1218,25 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size; + if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != + sizeof(header)) { + pj_ctx_fclose(ctx, fp); + return nullptr; + } + pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + + if (IsTIFF(header_size, header)) { + auto set = GTiffVGridShiftSet::open(ctx, fp, filename); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; + } + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); pj_ctx_fclose(ctx, fp); return nullptr; @@ -782,6 +1742,270 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, // --------------------------------------------------------------------------- +class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { + + GTiffHGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + + public: + ~GTiffHGridShiftSet() override; + + static std::unique_ptr + open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); +}; + +// --------------------------------------------------------------------------- + +class GTiffHGrid : public HorizontalShiftGrid { + friend void insertIntoHierarchy( + PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids); + + std::unique_ptr m_grid; + uint16 m_idxLatShift; + uint16 m_idxLonShift; + double m_convFactorToRadian; + bool m_positiveEast; + + public: + GTiffHGrid(std::unique_ptr &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast); + + ~GTiffHGrid() override; + + bool valueAt(int x, int y, float &lonShift, float &latShift) const override; + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); +}; + +// --------------------------------------------------------------------------- + +GTiffHGridShiftSet::~GTiffHGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffHGrid::GTiffHGrid(std::unique_ptr &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast) + : HorizontalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + m_grid(std::move(grid)), m_idxLatShift(idxLatShift), + m_idxLonShift(idxLonShift), m_convFactorToRadian(convFactorToRadian), + m_positiveEast(positiveEast) {} + +// --------------------------------------------------------------------------- + +GTiffHGrid::~GTiffHGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffHGrid::valueAt(int x, int y, float &lonShift, float &latShift) const { + if (!m_grid->valueAt(m_idxLatShift, x, y, latShift) || + !m_grid->valueAt(m_idxLonShift, x, y, lonShift)) { + return false; + } + // From arc-seconds to radians + latShift = static_cast(latShift * m_convFactorToRadian); + lonShift = static_cast(lonShift * m_convFactorToRadian); + if (!m_positiveEast) { + lonShift = -lonShift; + } + return true; +} + +// --------------------------------------------------------------------------- + +void GTiffHGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr +GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &filename) { + auto set = + std::unique_ptr(new GTiffHGridShiftSet(ctx, fp)); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + + // Defaults inspired from NTv2 + uint16 idxLatShift = 0; + uint16 idxLonShift = 1; + constexpr double ARC_SECOND_TO_RADIAN = (M_PI / 180.0) / 3600.0; + double convFactorToRadian = ARC_SECOND_TO_RADIAN; + bool positiveEast = true; + + std::map mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + if (grid->samplesPerPixel() < 2) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, + "At least 2 samples per pixel needed"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has not at least 2 samples", ifd); + continue; + } + } + + // Identify the index of the latitude and longitude offset channels + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForLatOffset = false; + bool foundDescriptionForLonOffset = false; + for (int i = 0; i < static_cast(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "latitude_offset") { + idxLatShift = static_cast(i); + foundDescriptionForLatOffset = true; + } else if (desc == "longitude_offset") { + idxLonShift = static_cast(i); + foundDescriptionForLonOffset = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + if (ifd > 0) { + // Assuming that extra IFD without + // longitude_offset/latitude_offset can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "longitude_offset/latitude_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "longitude_offset/latitude_offset channel"); + return nullptr; + } + } + } + if (foundDescriptionForLatOffset && !foundDescriptionForLonOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found latitude_offset channel, but not longitude_offset"); + return nullptr; + } else if (foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found longitude_offset channel, but not latitude_offset"); + return nullptr; + } + + if (idxLatShift >= grid->samplesPerPixel() || + idxLonShift >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + if (foundDescriptionForLonOffset) { + const std::string positiveValue = + grid->metadataItem("positive_value", idxLonShift); + if (!positiveValue.empty()) { + if (positiveValue == "west") { + positiveEast = false; + } else if (positiveValue == "east") { + positiveEast = true; + } else { + pj_log(ctx, PJ_LOG_ERROR, + "Unsupported value %s for 'positive_value'", + positiveValue.c_str()); + return nullptr; + } + } + } + + // Identify their unit + { + const auto unitLatShift = + grid->metadataItem("UNITTYPE", idxLatShift); + const auto unitLonShift = + grid->metadataItem("UNITTYPE", idxLonShift); + if (unitLatShift != unitLonShift) { + pj_log(ctx, PJ_LOG_ERROR, + "Different unit for longitude and latitude offset"); + return nullptr; + } + if (!unitLatShift.empty()) { + if (unitLatShift == "arc-second") { + convFactorToRadian = ARC_SECOND_TO_RADIAN; + } else if (unitLatShift == "radian") { + convFactorToRadian = 1.0; + } else if (unitLatShift == "degree") { + convFactorToRadian = M_PI / 180.0; + } else { + pj_log(ctx, PJ_LOG_ERROR, "Unsupported unit %s", + unitLatShift.c_str()); + return nullptr; + } + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_name"); + + auto hgrid = internal::make_unique( + std::move(grid), idxLatShift, idxLonShift, convFactorToRadian, + positiveEast); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} + +// --------------------------------------------------------------------------- + std::unique_ptr HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (filename == "null") { @@ -849,6 +2073,12 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { strncmp(header + 0, "NUM_OREC", 8) == 0 && strncmp(header + 48, "GS_TYPE", 7) == 0) { return NTv2GridSet::open(ctx, fp, filename); + } else if (IsTIFF(header_size, + reinterpret_cast(header))) { + auto set = GTiffHGridShiftSet::open(ctx, fp, filename); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); diff --git a/src/grids.hpp b/src/grids.hpp index 65bf502b..e40528c3 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -45,6 +45,8 @@ struct ExtentAndRes { double resLat; // in radian bool fullWorldLongitude() const; + bool contains(const ExtentAndRes &other) const; + bool intersects(const ExtentAndRes &other) const; }; // --------------------------------------------------------------------------- @@ -87,6 +89,7 @@ class VerticalShiftGrid : public Grid { // --------------------------------------------------------------------------- class VerticalShiftGridSet { + protected: std::string m_name{}; std::string m_format{}; std::vector> m_grids{}; -- cgit v1.2.3 From 2804ee5ec0982a91d7fdb1304bd1490ebb9643a9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 10 Dec 2019 19:42:38 +0100 Subject: GTiff grid: use a block cache per dataset for faster access --- src/grids.cpp | 164 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 44 deletions(-) (limited to 'src') diff --git a/src/grids.cpp b/src/grids.cpp index 010ddb22..471ef590 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -29,9 +29,11 @@ #ifndef FROM_PROJ_CPP #define FROM_PROJ_CPP #endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include "grids.hpp" #include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" #include "proj_internal.h" #include "tiffio.h" @@ -294,9 +296,67 @@ constexpr uint16 TIFFTAG_GDAL_NODATA = 42113; // --------------------------------------------------------------------------- +class BlockCache { + public: + void insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector &data); + std::shared_ptr> get(uint32 ifdIdx, + uint32 blockNumber); + + private: + struct Key { + uint32 ifdIdx; + uint32 blockNumber; + + Key(uint32 ifdIdxIn, uint32 blockNumberIn) + : ifdIdx(ifdIdxIn), blockNumber(blockNumberIn) {} + bool operator==(const Key &other) const { + return ifdIdx == other.ifdIdx && blockNumber == other.blockNumber; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return k.ifdIdx ^ (k.blockNumber << 16) ^ (k.blockNumber >> 16); + } + }; + + static constexpr int NUM_BLOCKS_AT_CROSSING_TILES = 4; + static constexpr int MAX_SAMPLE_COUNT = 3; + lru11::Cache< + Key, std::shared_ptr>, lru11::NullLock, + std::unordered_map< + Key, + typename std::list>>>::iterator, + KeyHasher>> + cache_{NUM_BLOCKS_AT_CROSSING_TILES * MAX_SAMPLE_COUNT}; +}; + +// --------------------------------------------------------------------------- + +void BlockCache::insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector &data) { + cache_.insert(Key(ifdIdx, blockNumber), + std::make_shared>(data)); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr> +BlockCache::get(uint32 ifdIdx, uint32 blockNumber) { + std::shared_ptr> ret; + cache_.tryGet(Key(ifdIdx, blockNumber), ret); + return ret; +} + +// --------------------------------------------------------------------------- + class GTiffGrid : public Grid { - PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset - TIFF *m_hTIFF; // owned by the belonging GTiffDataset + PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset + TIFF *m_hTIFF; // owned by the belonging GTiffDataset + BlockCache &m_cache; // owned by the belonging GTiffDataset + uint32 m_ifdIdx; TIFFDataType m_dt; uint16 m_samplesPerPixel; uint16 m_planarConfig; @@ -305,9 +365,6 @@ class GTiffGrid : public Grid { bool m_tiled; uint32 m_blockWidth = 0; uint32 m_blockHeight = 0; - mutable int m_lastSample = -1; - mutable int m_lastBlockX = -1; - mutable int m_lastBlockY = -1; mutable std::vector m_buffer{}; unsigned m_blocksPerRow = 0; unsigned m_blocksPerCol = 0; @@ -324,12 +381,14 @@ class GTiffGrid : public Grid { void getScaleOffset(double &scale, double &offset, uint16 sample) const; template - float readValue(uint32 offsetInBlock, uint16 sample) const; + float readValue(const std::vector &buffer, + uint32 offsetInBlock, uint16 sample) const; public: - GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, - const ExtentAndRes &extentIn, TIFFDataType dtIn, - uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx, + int widthIn, int heightIn, const ExtentAndRes &extentIn, + TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, + bool bottomUpIn); ~GTiffGrid() override; @@ -346,11 +405,13 @@ class GTiffGrid : public Grid { // --------------------------------------------------------------------------- -GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, + uint32 ifdIdx, int widthIn, int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn) - : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), m_dt(dtIn), + : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), + m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn), m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), m_tiled(TIFFIsTiled(hTIFF) != 0) { @@ -473,9 +534,10 @@ void GTiffGrid::getScaleOffset(double &scale, double &offset, // --------------------------------------------------------------------------- template -float GTiffGrid::readValue(uint32 offsetInBlock, uint16 sample) const { - const auto ptr = reinterpret_cast(m_buffer.data()); - assert(offsetInBlock < m_buffer.size() / sizeof(T)); +float GTiffGrid::readValue(const std::vector &buffer, + uint32 offsetInBlock, uint16 sample) const { + const auto ptr = reinterpret_cast(buffer.data()); + assert(offsetInBlock < buffer.size() / sizeof(T)); const auto val = ptr[offsetInBlock]; if (!m_hasNodata || static_cast(val) != m_noData) { double scale = 1; @@ -493,20 +555,6 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, float &out) const { assert(x >= 0 && yFromBottom >= 0 && x < m_width && yFromBottom < m_height); assert(sample < m_samplesPerPixel); - if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && - !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { - return false; - } - if (m_buffer.empty()) { - const auto blockSize = static_cast( - m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); - try { - m_buffer.resize(blockSize); - } catch (const std::exception &e) { - pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); - return false; - } - } const int blockX = x / m_blockWidth; @@ -518,12 +566,32 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, const int yTIFF = m_bottomUp ? yFromBottom : m_height - 1 - yFromBottom; const int blockY = yTIFF / m_blockHeight; - if (m_lastSample != static_cast(sample) || m_lastBlockX != blockX || - m_lastBlockY != blockY) { - uint32 blockId = blockY * m_blocksPerRow + blockX; - if (m_planarConfig == PLANARCONFIG_SEPARATE) { - blockId += sample * m_blocksPerCol * m_blocksPerRow; + uint32 blockId = blockY * m_blocksPerRow + blockX; + if (m_planarConfig == PLANARCONFIG_SEPARATE) { + blockId += sample * m_blocksPerCol * m_blocksPerRow; + } + + auto cachedBuffer = m_cache.get(m_ifdIdx, blockId); + std::vector *pBuffer = &m_buffer; + if (cachedBuffer != nullptr) { + // Safe as we don't access the cache before pBuffer is used + pBuffer = cachedBuffer.get(); + } else { + if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && + !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { + return false; } + if (m_buffer.empty()) { + const auto blockSize = static_cast( + m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); + try { + m_buffer.resize(blockSize); + } catch (const std::exception &e) { + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + return false; + } + } + if (m_tiled) { if (TIFFReadEncodedTile(m_hTIFF, blockId, m_buffer.data(), m_buffer.size()) == -1) { @@ -535,10 +603,15 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, return false; } } - m_lastSample = sample; - m_lastBlockX = blockX; - m_lastBlockY = blockY; + + try { + m_cache.insert(m_ifdIdx, blockId, m_buffer); + } catch (const std::exception &e) { + // Should normally not happen + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + } } + uint32 offsetInBlock = (x % m_blockWidth) + (yTIFF % m_blockHeight) * m_blockWidth; if (m_planarConfig == PLANARCONFIG_CONTIG) @@ -546,27 +619,27 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, switch (m_dt) { case TIFFDataType::Int16: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::UInt16: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::Int32: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::UInt32: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::Float32: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::Float64: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; } @@ -596,8 +669,10 @@ class GTiffDataset { PAFile m_fp; TIFF *m_hTIFF = nullptr; bool m_hasNextGrid = false; + uint32 m_ifdIdx = 0; toff_t m_nextDirOffset = 0; std::string m_filename{}; + BlockCache m_cache{}; GTiffDataset(const GTiffDataset &) = delete; GTiffDataset &operator=(const GTiffDataset &) = delete; @@ -948,8 +1023,9 @@ std::unique_ptr GTiffDataset::nextGrid() { } auto ret = std::unique_ptr( - new GTiffGrid(m_ctx, m_hTIFF, width, height, extent, dt, - samplesPerPixel, planarConfig, vRes < 0)); + new GTiffGrid(m_ctx, m_hTIFF, m_cache, m_ifdIdx, width, height, extent, + dt, samplesPerPixel, planarConfig, vRes < 0)); + m_ifdIdx++; m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); return ret; -- cgit v1.2.3 From e872984b46145fd03016fc53c6dac5843ba344f8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Dec 2019 16:41:06 +0100 Subject: grids.hpp: add a GenericShiftGridSet and GenericShiftGrid classes, that are going to be used by later commit --- src/grids.cpp | 299 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/grids.hpp | 53 +++++++++++ 2 files changed, 352 insertions(+) (limited to 'src') diff --git a/src/grids.cpp b/src/grids.cpp index 471ef590..eca0033a 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -2201,4 +2201,303 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, return nullptr; } +// --------------------------------------------------------------------------- + +class GTiffGenericGridShiftSet : public GenericShiftGridSet, + public GTiffDataset { + + GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) + : GTiffDataset(ctx, fp) {} + + public: + ~GTiffGenericGridShiftSet() override; + + static std::unique_ptr + open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); +}; + +// --------------------------------------------------------------------------- + +class GTiffGenericGrid : public GenericShiftGrid { + friend void insertIntoHierarchy( + PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids); + + std::unique_ptr m_grid; + + public: + GTiffGenericGrid(std::unique_ptr &&grid); + + ~GTiffGenericGrid() override; + + bool valueAt(int x, int y, int sample, float &out) const override; + + int samplesPerPixel() const override { return m_grid->samplesPerPixel(); } + + std::string unit(int sample) const override { + return m_grid->metadataItem("UNITTYPE", sample); + } + + std::string description(int sample) const override { + return m_grid->metadataItem("DESCRIPTION", sample); + } + + std::string metadataItem(const std::string &key, + int sample = -1) const override { + return m_grid->metadataItem(key, sample); + } + + void insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid); +}; + +// --------------------------------------------------------------------------- + +GTiffGenericGridShiftSet::~GTiffGenericGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::GTiffGenericGrid(std::unique_ptr &&grid) + : GenericShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + m_grid(std::move(grid)) {} + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::~GTiffGenericGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffGenericGrid::valueAt(int x, int y, int sample, float &out) const { + if (sample < 0 || + static_cast(sample) >= m_grid->samplesPerPixel()) + return false; + return m_grid->valueAt(static_cast(sample), x, y, out); +} + +// --------------------------------------------------------------------------- + +void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +class NullGenericShiftGrid : public GenericShiftGrid { + + public: + NullGenericShiftGrid() : GenericShiftGrid(3, 3, globalExtent()) {} + + bool isNullGrid() const override { return true; } + bool valueAt(int, int, int, float &out) const override; + + int samplesPerPixel() const override { return 0; } + + std::string unit(int) const override { return std::string(); } + + std::string description(int) const override { return std::string(); } + + std::string metadataItem(const std::string &, int) const override { + return std::string(); + } +}; + +// --------------------------------------------------------------------------- + +bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { + out = 0.0f; + return true; +} + +// --------------------------------------------------------------------------- + +std::unique_ptr +GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &filename) { + auto set = std::unique_ptr( + new GTiffGenericGridShiftSet(ctx, fp)); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + + std::map mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_name"); + + auto hgrid = internal::make_unique(std::move(grid)); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} + +// --------------------------------------------------------------------------- + +GenericShiftGrid::GenericShiftGrid(int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : Grid(widthIn, heightIn, extentIn) {} + +// --------------------------------------------------------------------------- + +GenericShiftGrid::~GenericShiftGrid() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::~GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +std::unique_ptr +GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { + if (filename == "null") { + auto set = + std::unique_ptr(new GenericShiftGridSet()); + set->m_name = filename; + set->m_format = "null"; + set->m_grids.push_back( + std::unique_ptr(new NullGenericShiftGrid())); + return set; + } + + PAFile fp; + if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + ctx->last_errno = 0; /* don't treat as a persistent error */ + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size; + if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != + sizeof(header)) { + pj_ctx_fclose(ctx, fp); + return nullptr; + } + pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + + if (IsTIFF(header_size, header)) { + auto set = GTiffGenericGridShiftSet::open(ctx, fp, filename); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; + } + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); + pj_ctx_fclose(ctx, fp); + return nullptr; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGrid::gridAt(double lon, double lat) const { + for (const auto &child : m_children) { + const auto &extentChild = child->extentAndRes(); + if ((extentChild.fullWorldLongitude() || + (lon >= extentChild.westLon && lon <= extentChild.eastLon)) && + lat >= extentChild.southLat && lat <= extentChild.northLat) { + return child->gridAt(lon, lat); + } + } + return this; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGridSet::gridAt(double lon, + double lat) const { + for (const auto &grid : m_grids) { + if (dynamic_cast(grid.get())) { + return grid.get(); + } + const auto &extent = grid->extentAndRes(); + if ((extent.fullWorldLongitude() || + (lon >= extent.westLon && lon <= extent.eastLon)) && + lat >= extent.southLat && lat <= extent.northLat) { + return grid->gridAt(lon, lat); + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { + std::string key("s"); + key += gridkey; + const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; + if (gridnames == nullptr) + return {}; + + auto listOfGridNames = internal::split(std::string(gridnames), ','); + ListOfGenericGrids grids; + for (const auto &gridnameStr : listOfGridNames) { + const char *gridname = gridnameStr.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = GenericShiftGridSet::open(P->ctx, gridname); + if (!gridSet) { + if (!canFail) { + pj_ctx_set_errno(P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return {}; + } + } else { + grids.emplace_back(std::move(gridSet)); + } + } + + return grids; +} + NS_PROJ_END diff --git a/src/grids.hpp b/src/grids.hpp index e40528c3..99683c39 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -154,11 +154,64 @@ class HorizontalShiftGridSet { // --------------------------------------------------------------------------- +class GenericShiftGrid : public Grid { + protected: + std::vector> m_children{}; + + public: + GenericShiftGrid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + + ~GenericShiftGrid() override; + + const GenericShiftGrid *gridAt(double lon, double lat) const; + + virtual std::string unit(int sample) const = 0; + + virtual std::string description(int sample) const = 0; + + virtual std::string metadataItem(const std::string &key, + int sample = -1) const = 0; + + virtual int samplesPerPixel() const = 0; + + // x = 0 is western-most column, y = 0 is southern-most line + virtual bool valueAt(int x, int y, int sample, float &out) const = 0; +}; + +// --------------------------------------------------------------------------- + +class GenericShiftGridSet { + protected: + std::string m_name{}; + std::string m_format{}; + std::vector> m_grids{}; + + GenericShiftGridSet(); + + public: + virtual ~GenericShiftGridSet(); + + static std::unique_ptr + open(PJ_CONTEXT *ctx, const std::string &filename); + + const std::string &name() const { return m_name; } + const std::string &format() const { return m_format; } + const std::vector> &grids() const { + return m_grids; + } + const GenericShiftGrid *gridAt(double lon, double lat) const; +}; + +// --------------------------------------------------------------------------- + typedef std::vector> ListOfHGrids; typedef std::vector> ListOfVGrids; +typedef std::vector> ListOfGenericGrids; ListOfVGrids proj_vgrid_init(PJ *P, const char *grids); ListOfHGrids proj_hgrid_init(PJ *P, const char *grids); +ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *grids); + double proj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, double vmultiplier); PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids &, PJ_LP lp); -- cgit v1.2.3 From 6d45bbc88bd58d7fc8a4ff7ec70bd2f23cb67aa1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Dec 2019 16:42:13 +0100 Subject: Add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid --- src/Makefile.am | 1 + src/lib_proj.cmake | 1 + src/pj_list.h | 1 + src/transformations/xyzgridshift.cpp | 329 +++++++++++++++++++++++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 src/transformations/xyzgridshift.cpp (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index f58e57fd..89f8dc0c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -183,6 +183,7 @@ libproj_la_SOURCES = \ transformations/horner.cpp \ transformations/molodensky.cpp \ transformations/vgridshift.cpp \ + transformations/xyzgridshift.cpp \ \ aasincos.cpp adjlon.cpp \ dmstor.cpp auth.cpp \ diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index f1547afe..f94f0562 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -198,6 +198,7 @@ set(SRC_LIBPROJ_TRANSFORMATIONS transformations/horner.cpp transformations/molodensky.cpp transformations/vgridshift.cpp + transformations/xyzgridshift.cpp ) set(SRC_LIBPROJ_ISO19111 diff --git a/src/pj_list.h b/src/pj_list.h index 0923bba8..9798a36b 100644 --- a/src/pj_list.h +++ b/src/pj_list.h @@ -170,3 +170,4 @@ PROJ_HEAD(weren, "Werenskiold I") PROJ_HEAD(wink1, "Winkel I") PROJ_HEAD(wink2, "Winkel II") PROJ_HEAD(wintri, "Winkel Tripel") +PROJ_HEAD(xyzgridshift, "XYZ grid shift") diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp new file mode 100644 index 00000000..f23c2588 --- /dev/null +++ b/src/transformations/xyzgridshift.cpp @@ -0,0 +1,329 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: Geocentric translation using a grid + * Author: Even Rouault, + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#define PJ_LIB__ + +#include +#include +#include +#include + +#include "proj_internal.h" +#include "grids.hpp" + +#include + +PROJ_HEAD(xyzgridshift, "Geocentric grid shift"); + +using namespace NS_PROJ; + +namespace { // anonymous namespace +struct xyzgridshiftData { + PJ *cart = nullptr; + bool grid_ref_is_input = true; + ListOfGenericGrids grids{}; + double multiplier = 1.0; +}; +} // anonymous namespace + + +// --------------------------------------------------------------------------- + +static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, + const PJ_LP& input) +{ + for( const auto& gridset: grids ) + { + auto grid = gridset->gridAt(input.lam, input.phi); + if( grid ) + return grid; + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static bool get_grid_values(PJ* P, + const xyzgridshiftData* Q, + const PJ_LP& lp, + double& dx, + double& dy, + double& dz) +{ + auto grid = findGrid(Q->grids, lp); + if( !grid ) { + return false; + } + if( grid->isNullGrid() ) { + dx = 0; + dy = 0; + dz = 0; + return true; + } + const auto samplesPerPixel = grid->samplesPerPixel(); + if( samplesPerPixel < 3 ) { + proj_log_error(P, "xyzgridshift: grid has not enough samples"); + return false; + } + int sampleX = 0; + int sampleY = 1; + int sampleZ = 2; + for( int i = 0; i < samplesPerPixel; i++ ) + { + const auto desc = grid->description(i); + if( desc == "x_translation") { + sampleX = i; + } else if( desc == "y_translation") { + sampleY = i; + } else if( desc == "z_translation") { + sampleZ = i; + } + } + const auto unit = grid->unit(sampleX); + if( !unit.empty() && unit != "metre" ) { + proj_log_error(P, "xyzgridshift: Only unit=metre currently handled"); + return false; + } + + const auto& extent = grid->extentAndRes(); + double grid_x = (lp.lam - extent.westLon) / extent.resLon; + double grid_y = (lp.phi - extent.southLat) / extent.resLat; + int ix = static_cast(grid_x); + int iy = static_cast(grid_y); + int ix2 = std::min(ix + 1, grid->width() - 1); + int iy2 = std::min(iy + 1, grid->height() - 1); + + float dx1, dy1, dz1; + if( !grid->valueAt(ix, iy, sampleX, dx1) || + !grid->valueAt(ix, iy, sampleY, dy1) || + !grid->valueAt(ix, iy, sampleZ, dz1) ) { + return false; + } + + float dx2, dy2, dz2; + if( !grid->valueAt(ix2, iy, sampleX, dx2) || + !grid->valueAt(ix2, iy, sampleY, dy2) || + !grid->valueAt(ix2, iy, sampleZ, dz2) ) { + return false; + } + + float dx3, dy3, dz3; + if( !grid->valueAt(ix, iy2, sampleX, dx3) || + !grid->valueAt(ix, iy2, sampleY, dy3) || + !grid->valueAt(ix, iy2, sampleZ, dz3) ) { + return false; + } + + float dx4, dy4, dz4; + if( !grid->valueAt(ix2, iy2, sampleX, dx4) || + !grid->valueAt(ix2, iy2, sampleY, dy4) || + !grid->valueAt(ix2, iy2, sampleZ, dz4) ) { + return false; + } + + double frct_lam = grid_x - ix; + double frct_phi = grid_y - iy; + double m10 = frct_lam; + double m11 = m10; + double m01 = 1. - frct_lam; + double m00 = m01; + m11 *= frct_phi; + m01 *= frct_phi; + frct_phi = 1. - frct_phi; + m00 *= frct_phi; + m10 *= frct_phi; + dx = (m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4) * Q->multiplier; + dy = (m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4) * Q->multiplier; + dz = (m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4) * Q->multiplier; + return true; +} + +// --------------------------------------------------------------------------- + +#define SQUARE(x) ((x)*(x)) + +// --------------------------------------------------------------------------- + +static PJ_COORD iterative_adjustment(PJ* P, + const xyzgridshiftData* Q, + const PJ_COORD& pointInit, + double factor) +{ + PJ_COORD point = pointInit; + for(int i = 0; i < 10; i++) { + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + + dx *= factor; + dy *= factor; + dz *= factor; + + const double err = SQUARE((point.xyz.x - pointInit.xyz.x) - dx) + + SQUARE((point.xyz.y - pointInit.xyz.y) - dy) + + SQUARE((point.xyz.z - pointInit.xyz.z) - dz); + + point.xyz.x = pointInit.xyz.x + dx; + point.xyz.y = pointInit.xyz.y + dy; + point.xyz.z = pointInit.xyz.z + dz; + if( err < 1e-10 ) { + break; + } + } + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_COORD direct_adjustment(PJ* P, + const xyzgridshiftData* Q, + PJ_COORD point, + double factor) +{ + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + point.xyz.x += factor * dx; + point.xyz.y += factor * dy; + point.xyz.z += factor * dz; + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { + auto Q = static_cast(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.lpz = lpz; + + if( Q->grid_ref_is_input ) { + point = direct_adjustment(P, Q, point, 1.0); + } + else { + point = iterative_adjustment(P, Q, point, 1.0); + } + + return point.xyz; +} + + +static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { + auto Q = static_cast(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.xyz = xyz; + + if( Q->grid_ref_is_input ) { + point = iterative_adjustment(P, Q, point, -1.0); + } + else { + point = direct_adjustment(P, Q, point, -1.0); + } + + return point.lpz; +} + +static PJ *destructor (PJ *P, int errlev) { + if (nullptr==P) + return nullptr; + + auto Q = static_cast(P->opaque); + if( Q ) + { + if (Q->cart) + Q->cart->destructor (Q->cart, errlev); + delete Q; + } + P->opaque = nullptr; + + return pj_default_destructor(P, errlev); +} + +PJ *TRANSFORMATION(xyzgridshift,0) { + auto Q = new xyzgridshiftData; + P->opaque = (void *) Q; + P->destructor = destructor; + + P->fwd4d = nullptr; + P->inv4d = nullptr; + P->fwd3d = forward_3d; + P->inv3d = reverse_3d; + P->fwd = nullptr; + P->inv = nullptr; + + P->left = PJ_IO_UNITS_CARTESIAN; + P->right = PJ_IO_UNITS_CARTESIAN; + + // Pass a dummy ellipsoid definition that will be overridden just afterwards + Q->cart = proj_create(P->ctx, "+proj=cart +a=1"); + if (Q->cart == nullptr) + return destructor(P, ENOMEM); + + /* inherit ellipsoid definition from P to Q->cart */ + pj_inherit_ellipsoid_def (P, Q->cart); + + const char* grid_ref = pj_param (P->ctx, P->params, "sgrid_ref").s; + if( grid_ref ) { + if (strcmp(grid_ref, "input_crs") == 0 ) { + // default + } else if (strcmp(grid_ref, "output_crs") == 0 ) { + // Convention use for example for NTF->RGF93 grid that contains + // delta x,y,z from NTF to RGF93, but the grid itself is referenced + // in RGF93 + Q->grid_ref_is_input = false; + } else { + proj_log_error(P, "xyzgridshift: unusupported value for grid_ref"); + return destructor (P, PJD_ERR_NO_ARGS); + } + } + + if (0==pj_param(P->ctx, P->params, "tgrids").i) { + proj_log_error(P, "xyzgridshift: +grids parameter missing."); + return destructor (P, PJD_ERR_NO_ARGS); + } + + /* multiplier for delta x,y,z */ + if (pj_param(P->ctx, P->params, "tmultiplier").i) { + Q->multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; + } + + Q->grids = proj_generic_grid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "xyzgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + + return P; +} -- cgit v1.2.3 From c165952bcf0135bc26b6cfea3ae387c51ba59fb0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 12 Dec 2019 19:27:57 +0100 Subject: grids.cpp: use 'parent_grid_name' as stated in RFC4 --- src/grids.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/grids.cpp b/src/grids.cpp index eca0033a..ca3ddfcc 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -1054,7 +1054,7 @@ insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr &&grid, std::map &mapGrids) { const auto &extent = grid->extentAndRes(); - // If we have one or both of grid_name and parent_name, try to use + // If we have one or both of grid_name and parent_grid_name, try to use // the names to recreate the hiearchy if (!gridName.empty()) { if (mapGrids.find(gridName) != mapGrids.end()) { @@ -1250,7 +1250,7 @@ GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } const std::string gridName = grid->metadataItem("grid_name"); - const std::string parentName = grid->metadataItem("parent_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); auto vgrid = internal::make_unique(std::move(grid), idxSample); @@ -2068,7 +2068,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } const std::string gridName = grid->metadataItem("grid_name"); - const std::string parentName = grid->metadataItem("parent_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); auto hgrid = internal::make_unique( std::move(grid), idxLatShift, idxLonShift, convFactorToRadian, @@ -2364,7 +2364,7 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } const std::string gridName = grid->metadataItem("grid_name"); - const std::string parentName = grid->metadataItem("parent_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); auto hgrid = internal::make_unique(std::move(grid)); -- cgit v1.2.3 From 49f8ea7ea4755144310722a927f1e6f788c39e51 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 13 Dec 2019 15:25:59 +0100 Subject: Add configure/CMake option, strongly discouraged, to disable TIFF support --- src/Makefile.am | 2 +- src/grids.cpp | 34 ++++++++++++++++++++++++++++++++++ src/lib_proj.cmake | 9 +++++++-- 3 files changed, 42 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index 89f8dc0c..9a4b3cae 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ + -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ diff --git a/src/grids.cpp b/src/grids.cpp index ca3ddfcc..090f7320 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -36,7 +36,9 @@ #include "proj/internal/lru_cache.hpp" #include "proj_internal.h" +#ifdef TIFF_ENABLED #include "tiffio.h" +#endif #include #include @@ -279,6 +281,8 @@ static bool IsTIFF(size_t header_size, const unsigned char *header) { (header[3] == 0x2B && header[2] == 0))); } +#ifdef TIFF_ENABLED + // --------------------------------------------------------------------------- enum class TIFFDataType { Int16, UInt16, Int32, UInt32, Float32, Float64 }; @@ -1044,6 +1048,8 @@ class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); }; +#endif // TIFF_ENABLED + // --------------------------------------------------------------------------- template @@ -1107,6 +1113,7 @@ insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr &&grid, } } +#ifdef TIFF_ENABLED // --------------------------------------------------------------------------- class GTiffVGrid : public VerticalShiftGrid { @@ -1260,6 +1267,7 @@ GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } return set; } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -1307,10 +1315,16 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_fseek(ctx, fp, SEEK_SET, 0); if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED auto set = GTiffVGridShiftSet::open(ctx, fp, filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; +#else + pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_ctx_fclose(ctx, fp); + return nullptr; +#endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); @@ -1816,6 +1830,8 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, return set; } +#ifdef TIFF_ENABLED + // --------------------------------------------------------------------------- class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { @@ -2079,6 +2095,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } return set; } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2151,10 +2168,16 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return NTv2GridSet::open(ctx, fp, filename); } else if (IsTIFF(header_size, reinterpret_cast(header))) { +#ifdef TIFF_ENABLED auto set = GTiffHGridShiftSet::open(ctx, fp, filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; +#else + pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_ctx_fclose(ctx, fp); + return nullptr; +#endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); @@ -2201,6 +2224,7 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, return nullptr; } +#ifdef TIFF_ENABLED // --------------------------------------------------------------------------- class GTiffGenericGridShiftSet : public GenericShiftGridSet, @@ -2298,6 +2322,7 @@ void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, m_children.emplace_back(std::move(subgrid)); } } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2329,6 +2354,8 @@ bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { // --------------------------------------------------------------------------- +#ifdef TIFF_ENABLED + std::unique_ptr GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename) { @@ -2373,6 +2400,7 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } return set; } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2425,10 +2453,16 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_fseek(ctx, fp, SEEK_SET, 0); if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED auto set = GTiffGenericGridShiftSet::open(ctx, fp, filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; +#else + pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_ctx_fclose(ctx, fp); + return nullptr; +#endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index f94f0562..d4c95092 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -436,8 +436,13 @@ if(USE_THREAD AND Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) target_link_libraries(${PROJ_CORE_TARGET} ${CMAKE_THREAD_LIBS_INIT}) endif() -include_directories(${SQLITE3_INCLUDE_DIR} ${TIFF_INCLUDE_DIR}) -target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY} ${TIFF_LIBRARY}) +include_directories(${SQLITE3_INCLUDE_DIR}) +target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY}) + +if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) + include_directories(${TIFF_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${TIFF_LIBRARY}) +endif() if(MSVC) target_compile_definitions(${PROJ_CORE_TARGET} -- cgit v1.2.3 From b934fd3885fa017b0c7a5ca59083a7f52fdaafc0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 14 Dec 2019 00:15:27 +0100 Subject: Grid class: add a name() method --- src/grids.cpp | 117 ++++++++++++++++++++++++++++++++++------------------------ src/grids.hpp | 13 +++++-- 2 files changed, 77 insertions(+), 53 deletions(-) (limited to 'src') diff --git a/src/grids.cpp b/src/grids.cpp index 090f7320..8fe6d745 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -95,8 +95,9 @@ bool ExtentAndRes::intersects(const ExtentAndRes &other) const { // --------------------------------------------------------------------------- -Grid::Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn) - : m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} +Grid::Grid(const std::string &name, int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : m_name(name), m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} // --------------------------------------------------------------------------- @@ -104,9 +105,9 @@ Grid::~Grid() = default; // --------------------------------------------------------------------------- -VerticalShiftGrid::VerticalShiftGrid(int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} +VerticalShiftGrid::VerticalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -132,7 +133,7 @@ static ExtentAndRes globalExtent() { class NullVerticalShiftGrid : public VerticalShiftGrid { public: - NullVerticalShiftGrid() : VerticalShiftGrid(3, 3, globalExtent()) {} + NullVerticalShiftGrid() : VerticalShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } bool valueAt(int, int, float &out) const override; @@ -156,17 +157,19 @@ class GTXVerticalShiftGrid : public VerticalShiftGrid { GTXVerticalShiftGrid &operator=(const GTXVerticalShiftGrid &) = delete; public: - GTXVerticalShiftGrid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, + GTXVerticalShiftGrid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, + int widthIn, int heightIn, const ExtentAndRes &extentIn) - : VerticalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), m_fp(fp) { - } + : VerticalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(fp) {} ~GTXVerticalShiftGrid() override; bool valueAt(int x, int y, float &out) const override; bool isNodata(float val, double multiplier) const override; - static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, PAFile fp); + static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &name); }; // --------------------------------------------------------------------------- @@ -175,7 +178,8 @@ GTXVerticalShiftGrid::~GTXVerticalShiftGrid() { pj_ctx_fclose(m_ctx, m_fp); } // --------------------------------------------------------------------------- -GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp) { +GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &name) { unsigned char header[40]; /* -------------------------------------------------------------------- */ @@ -231,7 +235,7 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp) { extent.eastLon = (xorigin + xstep * (columns - 1)) * DEG_TO_RAD; extent.northLat = (yorigin + ystep * (rows - 1)) * DEG_TO_RAD; - return new GTXVerticalShiftGrid(ctx, fp, columns, rows, extent); + return new GTXVerticalShiftGrid(ctx, fp, name, columns, rows, extent); } // --------------------------------------------------------------------------- @@ -390,9 +394,9 @@ class GTiffGrid : public Grid { public: GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx, - int widthIn, int heightIn, const ExtentAndRes &extentIn, - TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, - bool bottomUpIn); + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); ~GTiffGrid() override; @@ -410,11 +414,11 @@ class GTiffGrid : public Grid { // --------------------------------------------------------------------------- GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, - uint32 ifdIdx, int widthIn, int heightIn, - const ExtentAndRes &extentIn, TIFFDataType dtIn, - uint16 samplesPerPixelIn, uint16 planarConfig, - bool bottomUpIn) - : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), + uint32 ifdIdx, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn, + TIFFDataType dtIn, uint16 samplesPerPixelIn, + uint16 planarConfig, bool bottomUpIn) + : Grid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn), m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), @@ -512,6 +516,11 @@ GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, } catch (const std::exception &) { } } + + auto oIter = m_metadata.find(std::pair(-1, "grid_name")); + if (oIter != m_metadata.end()) { + m_name += ", " + oIter->second; + } } // --------------------------------------------------------------------------- @@ -1026,9 +1035,9 @@ std::unique_ptr GTiffDataset::nextGrid() { return nullptr; } - auto ret = std::unique_ptr( - new GTiffGrid(m_ctx, m_hTIFF, m_cache, m_ifdIdx, width, height, extent, - dt, samplesPerPixel, planarConfig, vRes < 0)); + auto ret = std::unique_ptr(new GTiffGrid( + m_ctx, m_hTIFF, m_cache, m_ifdIdx, m_filename, width, height, extent, + dt, samplesPerPixel, planarConfig, vRes < 0)); m_ifdIdx++; m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); @@ -1149,7 +1158,8 @@ GTiffVGridShiftSet::~GTiffVGridShiftSet() = default; // --------------------------------------------------------------------------- GTiffVGrid::GTiffVGrid(std::unique_ptr &&grid, uint16 idxSample) - : VerticalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + : VerticalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), m_grid(std::move(grid)), m_idxSample(idxSample) {} // --------------------------------------------------------------------------- @@ -1289,7 +1299,7 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return nullptr; } if (ends_with(filename, "gtx") || ends_with(filename, "GTX")) { - auto grid = GTXVerticalShiftGrid::open(ctx, fp); + auto grid = GTXVerticalShiftGrid::open(ctx, fp, filename); if (!grid) { pj_ctx_fclose(ctx, fp); return nullptr; @@ -1321,7 +1331,8 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else - pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); pj_ctx_fclose(ctx, fp); return nullptr; #endif @@ -1366,9 +1377,10 @@ const VerticalShiftGrid *VerticalShiftGridSet::gridAt(double lon, // --------------------------------------------------------------------------- -HorizontalShiftGrid::HorizontalShiftGrid(int widthIn, int heightIn, +HorizontalShiftGrid::HorizontalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -1387,7 +1399,8 @@ HorizontalShiftGridSet::~HorizontalShiftGridSet() = default; class NullHorizontalShiftGrid : public HorizontalShiftGrid { public: - NullHorizontalShiftGrid() : HorizontalShiftGrid(3, 3, globalExtent()) {} + NullHorizontalShiftGrid() + : HorizontalShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } @@ -1421,9 +1434,9 @@ class NTv1Grid : public HorizontalShiftGrid { NTv1Grid &operator=(const NTv1Grid &) = delete; public: - NTv1Grid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), + NTv1Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_fp(fp) {} ~NTv1Grid() override; @@ -1496,7 +1509,7 @@ NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, const int rows = static_cast( fabs((extent.northLat - extent.southLat) / extent.resLat + 0.5) + 1); - return new NTv1Grid(ctx, fp, columns, rows, extent); + return new NTv1Grid(ctx, fp, filename, columns, rows, extent); } // --------------------------------------------------------------------------- @@ -1535,9 +1548,9 @@ class CTable2Grid : public HorizontalShiftGrid { CTable2Grid &operator=(const CTable2Grid &) = delete; public: - CTable2Grid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), + CTable2Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, + int widthIn, int heightIn, const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_fp(fp) {} ~CTable2Grid() override; @@ -1602,7 +1615,7 @@ CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, extent.eastLon = extent.westLon + (width - 1) * extent.resLon; extent.northLat = extent.southLat + (height - 1) * extent.resLon; - return new CTable2Grid(ctx, fp, width, height, extent); + return new CTable2Grid(ctx, fp, filename, width, height, extent); } // --------------------------------------------------------------------------- @@ -1665,8 +1678,9 @@ class NTv2Grid : public HorizontalShiftGrid { NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, PAFile fp, unsigned long long offsetIn, bool mustSwapIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_name(nameIn), - m_ctx(ctx), m_fp(fp), m_offset(offsetIn), m_mustSwap(mustSwapIn) {} + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), + m_name(nameIn), m_ctx(ctx), m_fp(fp), m_offset(offsetIn), + m_mustSwap(mustSwapIn) {} bool valueAt(int, int, float &lonShift, float &latShift) const override; }; @@ -1811,8 +1825,9 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, } auto offset = pj_ctx_ftell(ctx, fp); - auto grid = std::unique_ptr(new NTv2Grid( - gridName, ctx, fp, offset, must_swap, columns, rows, extent)); + auto grid = std::unique_ptr( + new NTv2Grid(filename + ", " + gridName, ctx, fp, offset, must_swap, + columns, rows, extent)); std::string parentName; parentName.assign(header + 24, 8); auto iter = mapGrids.find(parentName); @@ -1881,7 +1896,8 @@ GTiffHGridShiftSet::~GTiffHGridShiftSet() = default; GTiffHGrid::GTiffHGrid(std::unique_ptr &&grid, uint16 idxLatShift, uint16 idxLonShift, double convFactorToRadian, bool positiveEast) - : HorizontalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + : HorizontalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), m_grid(std::move(grid)), m_idxLatShift(idxLatShift), m_idxLonShift(idxLonShift), m_convFactorToRadian(convFactorToRadian), m_positiveEast(positiveEast) {} @@ -2174,7 +2190,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else - pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); pj_ctx_fclose(ctx, fp); return nullptr; #endif @@ -2284,7 +2301,8 @@ GTiffGenericGridShiftSet::~GTiffGenericGridShiftSet() = default; // --------------------------------------------------------------------------- GTiffGenericGrid::GTiffGenericGrid(std::unique_ptr &&grid) - : GenericShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + : GenericShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), m_grid(std::move(grid)) {} // --------------------------------------------------------------------------- @@ -2329,7 +2347,7 @@ void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, class NullGenericShiftGrid : public GenericShiftGrid { public: - NullGenericShiftGrid() : GenericShiftGrid(3, 3, globalExtent()) {} + NullGenericShiftGrid() : GenericShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } bool valueAt(int, int, int, float &out) const override; @@ -2404,9 +2422,9 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, // --------------------------------------------------------------------------- -GenericShiftGrid::GenericShiftGrid(int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} +GenericShiftGrid::GenericShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -2459,7 +2477,8 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else - pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); pj_ctx_fclose(ctx, fp); return nullptr; #endif diff --git a/src/grids.hpp b/src/grids.hpp index 99683c39..0f595754 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -53,11 +53,13 @@ struct ExtentAndRes { class Grid { protected: + std::string m_name; int m_width; int m_height; ExtentAndRes m_extent; - Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + Grid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); public: virtual ~Grid(); @@ -65,6 +67,7 @@ class Grid { int width() const { return m_width; } int height() const { return m_height; } const ExtentAndRes &extentAndRes() const { return m_extent; } + const std::string &name() const { return m_name; } virtual bool isNullGrid() const { return false; } }; @@ -76,7 +79,8 @@ class VerticalShiftGrid : public Grid { std::vector> m_children{}; public: - VerticalShiftGrid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + VerticalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); const VerticalShiftGrid *gridAt(double lon, double lat) const; @@ -117,7 +121,7 @@ class HorizontalShiftGrid : public Grid { std::vector> m_children{}; public: - HorizontalShiftGrid(int widthIn, int heightIn, + HorizontalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn); ~HorizontalShiftGrid() override; @@ -159,7 +163,8 @@ class GenericShiftGrid : public Grid { std::vector> m_children{}; public: - GenericShiftGrid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + GenericShiftGrid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); ~GenericShiftGrid() override; -- cgit v1.2.3 From f36d2374268608d276da45c96a7b4792ab8ffdb5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 14 Dec 2019 00:16:14 +0100 Subject: Horizontal grid shift: fix issue on iterative inverse computation when switching between (sub)grids (fixes #1663) Given in.txt with 53.999759140 5.144478208 252.6995 Before the fix, cct -t 0 -d 4 +proj=pipeline +step +proj=axisswap +order=2,1,3,4 +step +proj=hgridshift +inv +grids=rdtrans2018.gsb +step +proj=vgridshift +grids=naptrans2018.gtx +step +proj=sterea +lat_0=52.156160556 +lon_0=5.387638889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel in.txt returned: 139079.8814 668306.0302 212.1724 0.0000 It now returns: 139079.8850 668306.0458 212.1724 0.0000 which meets with the 1mm accuracy the expected result of test point ``` 30010049 53.999759140 5.144478208 252.6995 139079.8850 668306.0460 212.1723 ``` --- src/apply_gridshift.cpp | 51 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp index c786a50a..b994474b 100644 --- a/src/apply_gridshift.cpp +++ b/src/apply_gridshift.cpp @@ -36,6 +36,8 @@ #include #include +#include + #include "proj.h" #include "proj_internal.h" #include "proj/internal/internal.hpp" @@ -191,7 +193,8 @@ static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { #define TOL 1e-12 static -PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { +PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* grid, + const ListOfHGrids& grids) { PJ_LP t, tb,del, dif; int i = MAX_ITERATIONS; const double toltol = TOL*TOL; @@ -201,9 +204,9 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { /* normalize input to ll origin */ tb = in; - const auto& extent = grid->extentAndRes(); - tb.lam -= extent.westLon; - tb.phi -= extent.southLat; + const auto* extent = &(grid->extentAndRes()); + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; tb.lam = adjlon (tb.lam - M_PI) + M_PI; @@ -223,17 +226,31 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { do { del = nad_intr(t, grid); - /* This case used to return failure, but I have - changed it to return the first order approximation - of the inverse shift. This avoids cases where the - grid shift *into* this grid came from another grid. - While we aren't returning optimally correct results - I feel a close result in this case is better than - no result. NFW - To demonstrate use -112.5839956 49.4914451 against - the NTv2 grid shift file from Canada. */ + /* We can possibly go outside of the initial guessed grid, so try */ + /* to fetch a new grid into which iterate... */ if (del.lam == HUGE_VAL) - break; + { + PJ_LP lp; + lp.lam = t.lam + extent->westLon; + lp.phi = t.phi + extent->southLat; + auto newGrid = findGrid(grids, lp); + if( newGrid == nullptr || newGrid == grid || newGrid->isNullGrid() ) + break; + pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", + grid->name().c_str(), + newGrid->name().c_str()); + grid = newGrid; + extent = &(grid->extentAndRes()); + t.lam = lp.lam - extent->westLon; + t.phi = lp.phi - extent->southLat; + tb = in; + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + tb.lam = adjlon (tb.lam - M_PI) + M_PI; + dif.lam = std::numeric_limits::max(); + dif.phi = std::numeric_limits::max(); + continue; + } dif.lam = t.lam + del.lam - tb.lam; dif.phi = t.phi + del.phi - tb.phi; @@ -254,8 +271,8 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { if (del.lam==HUGE_VAL && getenv ("PROJ_DEBUG")) fprintf (stderr, "Inverse grid shift iteration failed, presumably at grid edge.\nUsing first approximation.\n"); - in.lam = adjlon (t.lam + extent.westLon); - in.phi = t.phi + extent.southLat; + in.lam = adjlon (t.lam + extent->westLon); + in.phi = t.phi + extent->southLat; return in; } @@ -311,7 +328,7 @@ PJ_LP proj_hgrid_apply_internal(PJ_CONTEXT* ctx, } int inverse = direction == PJ_FWD ? 0 : 1; - out = nad_cvt(lp, inverse, grid); + out = nad_cvt(ctx, lp, inverse, grid, grids); if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); -- cgit v1.2.3 From a70e477087ff05ab24a54b806d89b5e4b970ec0a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2019 21:00:00 +0100 Subject: Rename PJ_CONTEXT::fileapi member as fileapi_legacy --- src/ctx.cpp | 8 ++++---- src/fileapi.cpp | 10 +++++----- src/proj_internal.h | 6 ++---- 3 files changed, 11 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/ctx.cpp b/src/ctx.cpp index bcb6e1cc..4536582d 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -95,7 +95,7 @@ projCtx_t projCtx_t::createDefault() projCtx_t ctx; ctx.debug_level = PJ_LOG_NONE; ctx.logger = pj_stderr_logger; - ctx.fileapi = pj_get_default_fileapi(); + ctx.fileapi_legacy = pj_get_default_fileapi(); if( getenv("PROJ_DEBUG") != nullptr ) { @@ -133,7 +133,7 @@ projCtx_t::projCtx_t(const projCtx_t& other) debug_level = other.debug_level; logger = other.logger; logger_app_data = other.logger_app_data; - fileapi = other.fileapi; + fileapi_legacy = other.fileapi_legacy; epsg_file_exists = other.epsg_file_exists; set_search_paths(other.search_paths); file_finder = other.file_finder; @@ -268,7 +268,7 @@ void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) { if (nullptr==ctx) return; - ctx->fileapi = fileapi; + ctx->fileapi_legacy = fileapi; } /************************************************************************/ @@ -280,6 +280,6 @@ projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) { if (nullptr==ctx) return nullptr; - return ctx->fileapi; + return ctx->fileapi_legacy; } diff --git a/src/fileapi.cpp b/src/fileapi.cpp index 70c7b5de..f39d94bb 100644 --- a/src/fileapi.cpp +++ b/src/fileapi.cpp @@ -141,7 +141,7 @@ static void stdio_fclose(PAFile file) PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) { - return ctx->fileapi->FOpen(ctx, filename, access); + return ctx->fileapi_legacy->FOpen(ctx, filename, access); } /************************************************************************/ @@ -149,7 +149,7 @@ PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) /************************************************************************/ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile file) { - return ctx->fileapi->FRead(buffer, size, nmemb, file); + return ctx->fileapi_legacy->FRead(buffer, size, nmemb, file); } /************************************************************************/ @@ -157,7 +157,7 @@ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile /************************************************************************/ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) { - return ctx->fileapi->FSeek(file, offset, whence); + return ctx->fileapi_legacy->FSeek(file, offset, whence); } /************************************************************************/ @@ -165,7 +165,7 @@ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) /************************************************************************/ long pj_ctx_ftell(projCtx ctx, PAFile file) { - return ctx->fileapi->FTell(file); + return ctx->fileapi_legacy->FTell(file); } /************************************************************************/ @@ -173,7 +173,7 @@ long pj_ctx_ftell(projCtx ctx, PAFile file) /************************************************************************/ void pj_ctx_fclose(projCtx ctx, PAFile file) { - ctx->fileapi->FClose(file); + ctx->fileapi_legacy->FClose(file); } /************************************************************************/ diff --git a/src/proj_internal.h b/src/proj_internal.h index 7d826414..93134946 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -214,9 +214,6 @@ size_t pj_trim_argc (char *args); char **pj_trim_argv (size_t argc, char *args); char *pj_make_args (size_t argc, char **argv); -/* Lowest level: Minimum support for fileapi */ -void proj_fileapi_set (PJ *P, void *fileapi); - typedef struct { double r, i; } COMPLEX; /* Forward declarations and typedefs for stuff needed inside the PJ object */ @@ -664,6 +661,7 @@ struct FACTORS { /* NOTE: Remember to update src/strerrno.cpp, src/apps/gie.cpp and transient_error in */ /* src/transform.cpp when adding new value */ +// Legacy struct projFileAPI_t; struct projCppContext; @@ -674,7 +672,7 @@ struct projCtx_t { int debug_level = 0; void (*logger)(void *, int, const char *) = nullptr; void *logger_app_data = nullptr; - struct projFileAPI_t *fileapi = nullptr; + struct projFileAPI_t *fileapi_legacy = nullptr; // for proj_api.h legacy API struct projCppContext* cpp_context = nullptr; /* internal context for C++ code */ int use_proj4_init_rules = -1; /* -1 = unknown, 0 = no, 1 = yes */ int epsg_file_exists = -1; /* -1 = unknown, 0 = no, 1 = yes */ -- cgit v1.2.3 From 2dfe2cdd1f4e963e6faaccd5ca29bc6d8fe4ae30 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2019 23:38:16 +0100 Subject: Add a FileManager and File class --- src/Makefile.am | 4 +- src/ctx.cpp | 25 -------- src/fileapi.cpp | 26 ++++++++ src/filemanager.cpp | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/filemanager.hpp | 68 +++++++++++++++++++++ src/lib_proj.cmake | 2 + src/open_lib.cpp | 65 ++++++++++++++++---- 7 files changed, 325 insertions(+), 38 deletions(-) create mode 100644 src/filemanager.cpp create mode 100644 src/filemanager.hpp (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index 9a4b3cae..f5cabe5e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,7 +215,9 @@ libproj_la_SOURCES = \ tracing.cpp \ \ grids.hpp \ - grids.cpp + grids.cpp \ + filemanager.hpp \ + filemanager.cpp # The sed hack is to please MSVC diff --git a/src/ctx.cpp b/src/ctx.cpp index 4536582d..da90d4d7 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -258,28 +258,3 @@ void *pj_ctx_get_app_data( projCtx ctx ) return nullptr; return ctx->logger_app_data; } - -/************************************************************************/ -/* pj_ctx_set_fileapi() */ -/************************************************************************/ - -void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) - -{ - if (nullptr==ctx) - return; - ctx->fileapi_legacy = fileapi; -} - -/************************************************************************/ -/* pj_ctx_get_fileapi() */ -/************************************************************************/ - -projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) - -{ - if (nullptr==ctx) - return nullptr; - return ctx->fileapi_legacy; -} - diff --git a/src/fileapi.cpp b/src/fileapi.cpp index f39d94bb..70be2502 100644 --- a/src/fileapi.cpp +++ b/src/fileapi.cpp @@ -34,6 +34,7 @@ #include "proj.h" #include "proj_internal.h" +#include "filemanager.hpp" static PAFile stdio_fopen(projCtx ctx, const char *filename, const char *access); @@ -212,3 +213,28 @@ char *pj_ctx_fgets(projCtx ctx, char *line, int size, PAFile file) } return line; } + +/************************************************************************/ +/* pj_ctx_set_fileapi() */ +/************************************************************************/ + +void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) + +{ + if (nullptr==ctx) + return; + ctx->fileapi_legacy = fileapi; +} + +/************************************************************************/ +/* pj_ctx_get_fileapi() */ +/************************************************************************/ + +projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) + +{ + if (nullptr==ctx) + return nullptr; + return ctx->fileapi_legacy; +} + diff --git a/src/filemanager.cpp b/src/filemanager.cpp new file mode 100644 index 00000000..b6164519 --- /dev/null +++ b/src/filemanager.cpp @@ -0,0 +1,173 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "filemanager.hpp" +#include "proj_internal.h" + +NS_PROJ_START + +// --------------------------------------------------------------------------- + +File::File() = default; + +// --------------------------------------------------------------------------- + +File::~File() = default; + +// --------------------------------------------------------------------------- + +class FileStdio : public File { + PJ_CONTEXT *m_ctx; + FILE *m_fp; + + FileStdio(const FileStdio &) = delete; + FileStdio &operator=(const FileStdio &) = delete; + + protected: + FileStdio(PJ_CONTEXT *ctx, FILE *fp) : m_ctx(ctx), m_fp(fp) {} + + public: + ~FileStdio() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileStdio::~FileStdio() { fclose(m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileStdio::read(void *buffer, size_t sizeBytes) { + return fread(buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileStdio::seek(unsigned long long offset, int whence) { + // TODO one day: use 64-bit offset compatible API + if (offset != static_cast(static_cast(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return fseek(m_fp, static_cast(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileStdio::tell() { + // TODO one day: use 64-bit offset compatible API + return ftell(m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr FileStdio::open(PJ_CONTEXT *ctx, const char *filename) { + auto fp = fopen(filename, "rb"); + return std::unique_ptr(fp ? new FileStdio(ctx, fp) : nullptr); +} + +// --------------------------------------------------------------------------- + +#ifndef REMOVE_LEGACY_SUPPORT + +class FileLegacyAdapter : public File { + PJ_CONTEXT *m_ctx; + PAFile m_fp; + + FileLegacyAdapter(const FileLegacyAdapter &) = delete; + FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; + + protected: + FileLegacyAdapter(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + + public: + ~FileLegacyAdapter() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileLegacyAdapter::~FileLegacyAdapter() { pj_ctx_fclose(m_ctx, m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileLegacyAdapter::read(void *buffer, size_t sizeBytes) { + return pj_ctx_fread(m_ctx, buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileLegacyAdapter::seek(unsigned long long offset, int whence) { + if (offset != static_cast(static_cast(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return pj_ctx_fseek(m_ctx, m_fp, static_cast(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileLegacyAdapter::tell() { + return pj_ctx_ftell(m_ctx, m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, + const char *filename) { + auto fid = pj_ctx_fopen(ctx, filename, "rb"); + return std::unique_ptr(fid ? new FileLegacyAdapter(ctx, fid) + : nullptr); +} + +#endif // REMOVE_LEGACY_SUPPORT + +// --------------------------------------------------------------------------- + +std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { +#ifndef REMOVE_LEGACY_SUPPORT + // If the user has specified a legacy fileapi, use it + if (ctx->fileapi_legacy != pj_get_default_fileapi()) { + return FileLegacyAdapter::open(ctx, filename); + } +#endif + return FileStdio::open(ctx, filename); +} + +NS_PROJ_END diff --git a/src/filemanager.hpp b/src/filemanager.hpp new file mode 100644 index 00000000..19478c48 --- /dev/null +++ b/src/filemanager.hpp @@ -0,0 +1,68 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FILEMANAGER_HPP_INCLUDED +#define FILEMANAGER_HPP_INCLUDED + +#include + +#include "proj.h" +#include "proj/util.hpp" + +NS_PROJ_START + +class File; + +class FileManager { + private: + FileManager() = delete; + + public: + // "Low-level" interface. + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); + + // "High-level" interface, honoring PROJ_LIB and the like. + static std::unique_ptr open_resource_file(PJ_CONTEXT *ctx, + const char *name); +}; + +// --------------------------------------------------------------------------- + +class File { + protected: + File(); + + public: + virtual ~File(); + virtual size_t read(void *buffer, size_t sizeBytes) = 0; + virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; + virtual unsigned long long tell() = 0; +}; + +NS_PROJ_END + +#endif // FILEMANAGER_HPP_INCLUDED \ No newline at end of file diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index d4c95092..eacc7a23 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -282,6 +282,8 @@ set(SRC_LIBPROJ_CORE tracing.cpp grids.hpp grids.cpp + filemanager.hpp + filemanager.cpp ${CMAKE_CURRENT_BINARY_DIR}/proj_config.h ) diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 24c31033..e1572754 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -44,6 +44,7 @@ #include "proj/internal/internal.hpp" #include "proj_internal.h" +#include "filemanager.hpp" static const char * proj_lib_name = #ifdef PROJ_LIB @@ -197,16 +198,17 @@ static const char *get_path_from_win32_projlib(const char *name, std::string& ou #endif /************************************************************************/ -/* pj_open_lib_ex() */ +/* pj_open_lib_internal() */ /************************************************************************/ -static PAFile -pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, - char* out_full_filename, size_t out_full_filename_size) { +static void* +pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, + void* (*open_file)(projCtx, const char*, const char*), + char* out_full_filename, size_t out_full_filename_size) { try { std::string fname; const char *sysname = nullptr; - PAFile fid = nullptr; + void* fid = nullptr; #ifdef WIN32 static const char dir_chars[] = "/\\"; const char dirSeparator = ';'; @@ -254,7 +256,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, fname += DIR_CHAR; fname += name; sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); + fid = open_file(ctx, sysname, mode); } catch( const std::exception& ) { } @@ -270,7 +272,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, fname += DIR_CHAR; fname += name; sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); + fid = open_file(ctx, sysname, mode); if( fid ) break; } @@ -290,7 +292,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, } assert(sysname); // to make Coverity Scan happy - if ( fid != nullptr || (fid = pj_ctx_fopen(ctx, sysname, mode)) != nullptr) + if ( fid != nullptr || (fid = open_file(ctx, sysname, mode)) != nullptr) { if( out_full_filename != nullptr && out_full_filename_size > 0 ) { @@ -321,19 +323,55 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, } } +/************************************************************************/ +/* pj_open_file_with_manager() */ +/************************************************************************/ + +static void* pj_open_file_with_manager(projCtx ctx, const char *name, + const char * /* mode */) +{ + return NS_PROJ::FileManager::open(ctx, name).release(); +} + +/************************************************************************/ +/* FileManager::open_resource_file() */ +/************************************************************************/ + +std::unique_ptr NS_PROJ::FileManager::open_resource_file( + projCtx ctx, const char *name) +{ + return std::unique_ptr( + reinterpret_cast( + pj_open_lib_internal(ctx, name, "rb", + pj_open_file_with_manager, + nullptr, 0))); +} + /************************************************************************/ /* pj_open_lib() */ /************************************************************************/ +#ifndef REMOVE_LEGACY_SUPPORT + +// Used by following legacy function +static void* pj_ctx_fopen_adapter(projCtx ctx, const char *name, const char *mode) +{ + return pj_ctx_fopen(ctx, name, mode); +} + +// Legacy function PAFile pj_open_lib(projCtx ctx, const char *name, const char *mode) { - return pj_open_lib_ex(ctx, name, mode, nullptr, 0); + return (PAFile)pj_open_lib_internal(ctx, name, mode, pj_ctx_fopen_adapter, nullptr, 0); } +#endif // REMOVE_LEGACY_SUPPORT + /************************************************************************/ /* pj_find_file() */ /************************************************************************/ + /** Returns the full filename corresponding to a proj resource file specified * as a short filename. * @@ -348,11 +386,14 @@ pj_open_lib(projCtx ctx, const char *name, const char *mode) { int pj_find_file(projCtx ctx, const char *short_filename, char* out_full_filename, size_t out_full_filename_size) { - PAFile f = pj_open_lib_ex(ctx, short_filename, "rb", out_full_filename, - out_full_filename_size); + auto f = reinterpret_cast( + pj_open_lib_internal(ctx, short_filename, "rb", + pj_open_file_with_manager, + out_full_filename, + out_full_filename_size)); if( f != nullptr ) { - pj_ctx_fclose(ctx, f); + delete f; return 1; } return 0; -- cgit v1.2.3 From cde119a2512341c07f775a8ee8f49c6920ec74f0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2019 23:53:09 +0100 Subject: grids.cpp: use FileManager/File interfaces --- src/grids.cpp | 241 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 115 insertions(+), 126 deletions(-) (limited to 'src') diff --git a/src/grids.cpp b/src/grids.cpp index 8fe6d745..91e51016 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -32,6 +32,7 @@ #define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include "grids.hpp" +#include "filemanager.hpp" #include "proj/internal/internal.hpp" #include "proj/internal/lru_cache.hpp" #include "proj_internal.h" @@ -151,41 +152,42 @@ bool NullVerticalShiftGrid::valueAt(int, int, float &out) const { class GTXVerticalShiftGrid : public VerticalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; GTXVerticalShiftGrid(const GTXVerticalShiftGrid &) = delete; GTXVerticalShiftGrid &operator=(const GTXVerticalShiftGrid &) = delete; public: - GTXVerticalShiftGrid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, - int widthIn, int heightIn, - const ExtentAndRes &extentIn) + explicit GTXVerticalShiftGrid(PJ_CONTEXT *ctx, std::unique_ptr &&fp, + const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) : VerticalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + m_fp(std::move(fp)) {} ~GTXVerticalShiftGrid() override; bool valueAt(int x, int y, float &out) const override; bool isNodata(float val, double multiplier) const override; - static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, PAFile fp, + static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &name); }; // --------------------------------------------------------------------------- -GTXVerticalShiftGrid::~GTXVerticalShiftGrid() { pj_ctx_fclose(m_ctx, m_fp); } +GTXVerticalShiftGrid::~GTXVerticalShiftGrid() = default; // --------------------------------------------------------------------------- -GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, +GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, + std::unique_ptr fp, const std::string &name) { unsigned char header[40]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -235,7 +237,8 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, extent.eastLon = (xorigin + xstep * (columns - 1)) * DEG_TO_RAD; extent.northLat = (yorigin + ystep * (rows - 1)) * DEG_TO_RAD; - return new GTXVerticalShiftGrid(ctx, fp, name, columns, rows, extent); + return new GTXVerticalShiftGrid(ctx, std::move(fp), name, columns, rows, + extent); } // --------------------------------------------------------------------------- @@ -243,8 +246,8 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, bool GTXVerticalShiftGrid::valueAt(int x, int y, float &out) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); - pj_ctx_fseek(m_ctx, m_fp, 40 + sizeof(float) * (y * m_width + x), SEEK_SET); - if (pj_ctx_fread(m_ctx, &out, sizeof(out), 1, m_fp) != 1) { + m_fp->seek(40 + sizeof(float) * (y * m_width + x)); + if (m_fp->read(&out, sizeof(out)) != sizeof(out)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -679,7 +682,7 @@ std::string GTiffGrid::metadataItem(const std::string &key, int sample) const { class GTiffDataset { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; TIFF *m_hTIFF = nullptr; bool m_hasNextGrid = false; uint32 m_ifdIdx = 0; @@ -693,7 +696,7 @@ class GTiffDataset { // libtiff I/O routines static tsize_t tiffReadProc(thandle_t fd, tdata_t buf, tsize_t size) { GTiffDataset *self = static_cast(fd); - return pj_ctx_fread(self->m_ctx, buf, 1, size, self->m_fp); + return self->m_fp->read(buf, size); } static tsize_t tiffWriteProc(thandle_t, tdata_t, tsize_t) { @@ -703,11 +706,8 @@ class GTiffDataset { static toff_t tiffSeekProc(thandle_t fd, toff_t off, int whence) { GTiffDataset *self = static_cast(fd); - // FIXME Remove cast to long when pj_ctx_fseek supports unsigned long - // long - if (pj_ctx_fseek(self->m_ctx, self->m_fp, static_cast(off), - whence) == 0) - return static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); + if (self->m_fp->seek(off, whence)) + return static_cast(self->m_fp->tell()); else return static_cast(-1); } @@ -719,11 +719,10 @@ class GTiffDataset { static toff_t tiffSizeProc(thandle_t fd) { GTiffDataset *self = static_cast(fd); - const auto old_off = pj_ctx_ftell(self->m_ctx, self->m_fp); - pj_ctx_fseek(self->m_ctx, self->m_fp, 0, SEEK_END); - const auto file_size = - static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); - pj_ctx_fseek(self->m_ctx, self->m_fp, old_off, SEEK_SET); + const auto old_off = self->m_fp->tell(); + self->m_fp->seek(0, SEEK_END); + const auto file_size = static_cast(self->m_fp->tell()); + self->m_fp->seek(old_off); return file_size; } @@ -732,7 +731,8 @@ class GTiffDataset { static void tiffUnmapProc(thandle_t, tdata_t, toff_t) {} public: - GTiffDataset(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + GTiffDataset(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : m_ctx(ctx), m_fp(std::move(fp)) {} virtual ~GTiffDataset(); bool openTIFF(const std::string &filename); @@ -745,7 +745,6 @@ class GTiffDataset { GTiffDataset::~GTiffDataset() { if (m_hTIFF) TIFFClose(m_hTIFF); - pj_ctx_fclose(m_ctx, m_fp); } // --------------------------------------------------------------------------- @@ -1048,13 +1047,15 @@ std::unique_ptr GTiffDataset::nextGrid() { class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { - GTiffVGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + GTiffVGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : GTiffDataset(ctx, std::move(fp)) {} public: ~GTiffVGridShiftSet() override; static std::unique_ptr - open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); + open(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &filename); }; #endif // TIFF_ENABLED @@ -1192,10 +1193,10 @@ void GTiffVGrid::insertGrid(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- std::unique_ptr -GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, +GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { - auto set = - std::unique_ptr(new GTiffVGridShiftSet(ctx, fp)); + auto set = std::unique_ptr( + new GTiffVGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; if (!set->openTIFF(filename)) { @@ -1293,15 +1294,14 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } if (ends_with(filename, "gtx") || ends_with(filename, "GTX")) { - auto grid = GTXVerticalShiftGrid::open(ctx, fp, filename); + auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), filename); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = @@ -1316,30 +1316,26 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { /* Load a header, to determine the file type. */ /* -------------------------------------------------------------------- */ unsigned char header[4]; - size_t header_size; - if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != - sizeof(header)) { - pj_ctx_fclose(ctx, fp); + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { return nullptr; } - pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + fp->seek(0); if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffVGridShiftSet::open(ctx, fp, filename); + auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); - pj_ctx_fclose(ctx, fp); return nullptr; #endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } @@ -1428,39 +1424,40 @@ static double to_double(const void *data) { class NTv1Grid : public HorizontalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; NTv1Grid(const NTv1Grid &) = delete; NTv1Grid &operator=(const NTv1Grid &) = delete; public: - NTv1Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, int widthIn, - int heightIn, const ExtentAndRes &extentIn) + explicit NTv1Grid(PJ_CONTEXT *ctx, std::unique_ptr &&fp, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn) : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + m_fp(std::move(fp)) {} ~NTv1Grid() override; bool valueAt(int, int, float &lonShift, float &latShift) const override; - static NTv1Grid *open(PJ_CONTEXT *ctx, PAFile fp, + static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); }; // --------------------------------------------------------------------------- -NTv1Grid::~NTv1Grid() { pj_ctx_fclose(m_ctx, m_fp); } +NTv1Grid::~NTv1Grid() = default; // --------------------------------------------------------------------------- -NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, +NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { unsigned char header[192]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1509,7 +1506,7 @@ NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, const int rows = static_cast( fabs((extent.northLat - extent.southLat) / extent.resLat + 0.5) + 1); - return new NTv1Grid(ctx, fp, filename, columns, rows, extent); + return new NTv1Grid(ctx, std::move(fp), filename, columns, rows, extent); } // --------------------------------------------------------------------------- @@ -1519,11 +1516,9 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { double two_doubles[2]; // NTv1 is organized from east to west ! - pj_ctx_fseek(m_ctx, m_fp, - 192 + 2 * sizeof(double) * (y * m_width + m_width - 1 - x), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_doubles[0], sizeof(two_doubles), 1, m_fp) != - 1) { + m_fp->seek(192 + 2 * sizeof(double) * (y * m_width + m_width - 1 - x)); + if (m_fp->read(&two_doubles[0], sizeof(two_doubles)) != + sizeof(two_doubles)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -1542,39 +1537,40 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { class CTable2Grid : public HorizontalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; CTable2Grid(const CTable2Grid &) = delete; CTable2Grid &operator=(const CTable2Grid &) = delete; public: - CTable2Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, - int widthIn, int heightIn, const ExtentAndRes &extentIn) + CTable2Grid(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn) : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + m_fp(std::move(fp)) {} ~CTable2Grid() override; bool valueAt(int, int, float &lonShift, float &latShift) const override; - static CTable2Grid *open(PJ_CONTEXT *ctx, PAFile fp, + static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); }; // --------------------------------------------------------------------------- -CTable2Grid::~CTable2Grid() { pj_ctx_fclose(m_ctx, m_fp); } +CTable2Grid::~CTable2Grid() = default; // --------------------------------------------------------------------------- -CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, +CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { unsigned char header[160]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1615,7 +1611,7 @@ CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, extent.eastLon = extent.westLon + (width - 1) * extent.resLon; extent.northLat = extent.southLat + (height - 1) * extent.resLon; - return new CTable2Grid(ctx, fp, filename, width, height, extent); + return new CTable2Grid(ctx, std::move(fp), filename, width, height, extent); } // --------------------------------------------------------------------------- @@ -1625,9 +1621,8 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_floats[2]; - pj_ctx_fseek(m_ctx, m_fp, 160 + 2 * sizeof(float) * (y * m_width + x), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_floats[0], sizeof(two_floats), 1, m_fp) != 1) { + m_fp->seek(160 + 2 * sizeof(float) * (y * m_width + x)); + if (m_fp->read(&two_floats[0], sizeof(two_floats)) != sizeof(two_floats)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -1645,18 +1640,18 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, // --------------------------------------------------------------------------- class NTv2GridSet : public HorizontalShiftGridSet { - PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; NTv2GridSet(const NTv2GridSet &) = delete; NTv2GridSet &operator=(const NTv2GridSet &) = delete; - NTv2GridSet(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + explicit NTv2GridSet(std::unique_ptr &&fp) : m_fp(std::move(fp)) {} public: ~NTv2GridSet() override; - static std::unique_ptr open(PJ_CONTEXT *ctx, PAFile fp, + static std::unique_ptr open(PJ_CONTEXT *ctx, + std::unique_ptr fp, const std::string &filename); }; @@ -1667,7 +1662,7 @@ class NTv2Grid : public HorizontalShiftGrid { std::string m_name; PJ_CONTEXT *m_ctx; // owned by the parent NTv2GridSet - PAFile m_fp; // owned by the parent NTv2GridSet + File *m_fp; // owned by the parent NTv2GridSet unsigned long long m_offset; bool m_mustSwap; @@ -1675,7 +1670,7 @@ class NTv2Grid : public HorizontalShiftGrid { NTv2Grid &operator=(const NTv2Grid &) = delete; public: - NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, PAFile fp, + NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, File *fp, unsigned long long offsetIn, bool mustSwapIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), @@ -1693,13 +1688,11 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { float two_float[2]; // NTv2 is organized from east to west ! // there are 4 components: lat shift, lon shift, lat error, lon error - pj_ctx_fseek( - m_ctx, m_fp, - // FIXME when fseek support unsigned long long - static_cast(m_offset + - 4 * sizeof(float) * (y * m_width + m_width - 1 - x)), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_float[0], sizeof(two_float), 1, m_fp) != 1) { + m_fp->seek( + m_offset + + 4 * sizeof(float) * + (static_cast(y) * m_width + m_width - 1 - x)); + if (m_fp->read(&two_float[0], sizeof(two_float)) != sizeof(two_float)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -1715,13 +1708,15 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { // --------------------------------------------------------------------------- -NTv2GridSet::~NTv2GridSet() { pj_ctx_fclose(m_ctx, m_fp); } +NTv2GridSet::~NTv2GridSet() = default; // --------------------------------------------------------------------------- -std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, +std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, + std::unique_ptr fp, const std::string &filename) { - auto set = std::unique_ptr(new NTv2GridSet(ctx, fp)); + File *fpRaw = fp.get(); + auto set = std::unique_ptr(new NTv2GridSet(std::move(fp))); set->m_name = filename; set->m_format = "ntv2"; @@ -1730,7 +1725,7 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1759,7 +1754,7 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, /* ==================================================================== */ for (unsigned subfile = 0; subfile < num_subfiles; subfile++) { // Read header - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1824,10 +1819,10 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, return nullptr; } - auto offset = pj_ctx_ftell(ctx, fp); + const auto offset = fpRaw->tell(); auto grid = std::unique_ptr( - new NTv2Grid(filename + ", " + gridName, ctx, fp, offset, must_swap, - columns, rows, extent)); + new NTv2Grid(filename + ", " + gridName, ctx, fpRaw, offset, + must_swap, columns, rows, extent)); std::string parentName; parentName.assign(header + 24, 8); auto iter = mapGrids.find(parentName); @@ -1840,7 +1835,8 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, mapGrids[gridName] = gridPtr; // Skip grid data. 4 components of size float - pj_ctx_fseek(ctx, fp, gs_count * 4 * 4, SEEK_CUR); + fpRaw->seek(static_cast(gs_count) * 4 * 4, + SEEK_CUR); } return set; } @@ -1851,13 +1847,15 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { - GTiffHGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + GTiffHGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : GTiffDataset(ctx, std::move(fp)) {} public: ~GTiffHGridShiftSet() override; static std::unique_ptr - open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); + open(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &filename); }; // --------------------------------------------------------------------------- @@ -1948,10 +1946,10 @@ void GTiffHGrid::insertGrid(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- std::unique_ptr -GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, +GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { - auto set = - std::unique_ptr(new GTiffHGridShiftSet(ctx, fp)); + auto set = std::unique_ptr( + new GTiffHGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; if (!set->openTIFF(filename)) { @@ -2127,8 +2125,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } @@ -2137,17 +2135,15 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { /* -------------------------------------------------------------------- */ /* Load a header, to determine the file type. */ /* -------------------------------------------------------------------- */ - size_t header_size; - if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != - sizeof(header)) { + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { /* some files may be smaller that sizeof(header), eg 160, so */ ctx->last_errno = 0; /* don't treat as a persistent error */ pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "pj_gridinfo_init: short header read of %d bytes", (int)header_size); } - - pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + fp->seek(0); /* -------------------------------------------------------------------- */ /* Determine file type. */ @@ -2155,9 +2151,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (header_size >= 144 + 16 && strncmp(header + 0, "HEADER", 6) == 0 && strncmp(header + 96, "W GRID", 6) == 0 && strncmp(header + 144, "TO NAD83 ", 16) == 0) { - auto grid = NTv1Grid::open(ctx, fp, filename); + auto grid = NTv1Grid::open(ctx, std::move(fp), filename); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = std::unique_ptr( @@ -2167,9 +2162,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { set->m_grids.push_back(std::unique_ptr(grid)); return set; } else if (header_size >= 9 && strncmp(header + 0, "CTABLE V2", 9) == 0) { - auto grid = CTable2Grid::open(ctx, fp, filename); + auto grid = CTable2Grid::open(ctx, std::move(fp), filename); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = std::unique_ptr( @@ -2181,24 +2175,22 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { } else if (header_size >= 48 + 7 && strncmp(header + 0, "NUM_OREC", 8) == 0 && strncmp(header + 48, "GS_TYPE", 7) == 0) { - return NTv2GridSet::open(ctx, fp, filename); + return NTv2GridSet::open(ctx, std::move(fp), filename); } else if (IsTIFF(header_size, reinterpret_cast(header))) { #ifdef TIFF_ENABLED - auto set = GTiffHGridShiftSet::open(ctx, fp, filename); + auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); - pj_ctx_fclose(ctx, fp); return nullptr; #endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } @@ -2247,14 +2239,15 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, class GTiffGenericGridShiftSet : public GenericShiftGridSet, public GTiffDataset { - GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) - : GTiffDataset(ctx, fp) {} + GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : GTiffDataset(ctx, std::move(fp)) {} public: ~GTiffGenericGridShiftSet() override; static std::unique_ptr - open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); + open(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &filename); }; // --------------------------------------------------------------------------- @@ -2375,10 +2368,10 @@ bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { #ifdef TIFF_ENABLED std::unique_ptr -GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, +GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { auto set = std::unique_ptr( - new GTiffGenericGridShiftSet(ctx, fp)); + new GTiffGenericGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; if (!set->openTIFF(filename)) { @@ -2452,8 +2445,8 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } @@ -2462,30 +2455,26 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { /* Load a header, to determine the file type. */ /* -------------------------------------------------------------------- */ unsigned char header[4]; - size_t header_size; - if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != - sizeof(header)) { - pj_ctx_fclose(ctx, fp); + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { return nullptr; } - pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + fp->seek(0); if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffGenericGridShiftSet::open(ctx, fp, filename); + auto set = GTiffGenericGridShiftSet::open(ctx, std::move(fp), filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); - pj_ctx_fclose(ctx, fp); return nullptr; #endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } -- cgit v1.2.3 From d6ae5289b603dae07e5204695a7735d86a9c1c1e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 16:25:58 +0100 Subject: Build: add optional curl dependency --- src/Makefile.am | 4 ++-- src/lib_proj.cmake | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/Makefile.am b/src/Makefile.am index f5cabe5e..afe4bcb7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ + -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ @@ -43,7 +43,7 @@ geodtest_LDADD = libproj.la lib_LTLIBRARIES = libproj.la libproj_la_LDFLAGS = -no-undefined -version-info 17:0:2 -libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ +libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ @CURL_LIBS@ libproj_la_SOURCES = \ pj_list.h proj_internal.h \ diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index eacc7a23..12dcb366 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -446,6 +446,11 @@ if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) target_link_libraries(${PROJ_CORE_TARGET} ${TIFF_LIBRARY}) endif() +if(CURL_FOUND) + include_directories(${CURL_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${CURL_LIBRARY}) +endif() + if(MSVC) target_compile_definitions(${PROJ_CORE_TARGET} PRIVATE PROJ_MSVC_DLL_EXPORT=1) -- cgit v1.2.3 From a06f4a258f618dbad2ce01feadab6908db00bda5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 21:24:53 +0100 Subject: Add proj_context_set_network_callbacks() with an empty implementation --- src/ctx.cpp | 3 + src/filemanager.cpp | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/filemanager.hpp | 6 ++ src/open_lib.cpp | 6 +- src/proj.h | 76 ++++++++++++++++++++++++ src/proj_internal.h | 13 +++++ 6 files changed, 267 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/ctx.cpp b/src/ctx.cpp index da90d4d7..a7cf8cb3 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -33,6 +33,7 @@ #include "proj_experimental.h" #include "proj_internal.h" +#include "filemanager.hpp" /************************************************************************/ /* pj_get_ctx() */ @@ -96,6 +97,7 @@ projCtx_t projCtx_t::createDefault() ctx.debug_level = PJ_LOG_NONE; ctx.logger = pj_stderr_logger; ctx.fileapi_legacy = pj_get_default_fileapi(); + NS_PROJ::FileManager::fillDefaultNetworkInterface(&ctx); if( getenv("PROJ_DEBUG") != nullptr ) { @@ -139,6 +141,7 @@ projCtx_t::projCtx_t(const projCtx_t& other) file_finder = other.file_finder; file_finder_legacy = other.file_finder_legacy; file_finder_user_data = other.file_finder_user_data; + networking = other.networking; } /************************************************************************/ diff --git a/src/filemanager.cpp b/src/filemanager.cpp index b6164519..97f6a4dc 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -25,11 +25,21 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ +#ifndef FROM_PROJ_CPP +#define FROM_PROJ_CPP +#endif + #include "filemanager.hpp" +#include "proj.h" +#include "proj/internal/internal.hpp" #include "proj_internal.h" +//! @cond Doxygen_Suppress + NS_PROJ_START +using namespace internal; + // --------------------------------------------------------------------------- File::File() = default; @@ -160,6 +170,79 @@ std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +class NetworkFile : public File { + PJ_CONTEXT *m_ctx; + PROJ_NETWORK_HANDLE *m_handle; + unsigned long long m_pos = 0; + + NetworkFile(const NetworkFile &) = delete; + NetworkFile &operator=(const NetworkFile &) = delete; + + protected: + NetworkFile(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle) + : m_ctx(ctx), m_handle(handle) {} + + public: + ~NetworkFile() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { + std::vector buffer(16 * 1024); + size_t size_read = 0; + auto handle = ctx->networking.open(ctx, filename, buffer.size(), &buffer[0], + &size_read, ctx->networking.user_data); + return std::unique_ptr(handle ? new NetworkFile(ctx, handle) + : nullptr); +} + +// --------------------------------------------------------------------------- + +size_t NetworkFile::read(void *buffer, size_t sizeBytes) { + size_t nRead = m_ctx->networking.read_range( + m_ctx, m_handle, m_pos, sizeBytes, buffer, m_ctx->networking.user_data); + m_pos += nRead; + return nRead; +} + +// --------------------------------------------------------------------------- + +bool NetworkFile::seek(unsigned long long offset, int whence) { + if (whence == SEEK_SET) { + m_pos = offset; + } else if (whence == SEEK_CUR) { + m_pos += offset; + } else { + if (offset != 0) + return false; + const auto filesize = m_ctx->networking.get_file_size( + m_ctx, m_handle, m_ctx->networking.user_data); + if (filesize == 0) + return false; + m_pos = filesize; + } + return true; +} + +// --------------------------------------------------------------------------- + +unsigned long long NetworkFile::tell() { return m_pos; } + +// --------------------------------------------------------------------------- + +NetworkFile::~NetworkFile() { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); +} + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifndef REMOVE_LEGACY_SUPPORT // If the user has specified a legacy fileapi, use it @@ -167,7 +250,88 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { return FileLegacyAdapter::open(ctx, filename); } #endif + if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + return NetworkFile::open(ctx, filename); + } return FileStdio::open(ctx, filename); } +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE * +no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ + void * /*user_data*/) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); + return nullptr; +} + +// --------------------------------------------------------------------------- + +static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, + void * /*user_data*/) {} + +// --------------------------------------------------------------------------- + +static const char *no_op_network_get_last_error(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *, + void * /*user_data*/) { + return "Network functionality not available"; +} + +// --------------------------------------------------------------------------- + +void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { + ctx->networking.open = no_op_network_open; + ctx->networking.close = no_op_network_close; + ctx->networking.get_last_error = no_op_network_get_last_error; +} + +// --------------------------------------------------------------------------- + NS_PROJ_END + +//! @endcond + +// --------------------------------------------------------------------------- + +/** Define a custom set of callbacks for network access. + * + * All callbacks should be provided (non NULL pointers). + * + * @param ctx PROJ context, or NULL + * @param open_cbk Callback to open a remote file given its URL + * @param close_cbk Callback to close a remote file. + * @param get_header_value_cbk Callback to get HTTP headers + * @param get_file_size_cbk Callback to get the size of the remote file. + * @param read_range_cbk Callback to read a range of bytes inside a remote file. + * @param get_last_error_cbk Callback to get last error message. + * @param user_data Arbitrary pointer provided by the user, and passed to the + * above callbacks. May be NULL. + * @return TRUE in case of success. + */ +int proj_context_set_network_callbacks( + PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_get_file_size_cbk_type get_file_size_cbk, + proj_network_read_range_type read_range_cbk, + proj_network_get_last_error_type get_last_error_cbk, void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!open_cbk || !close_cbk || !get_header_value_cbk || + !get_file_size_cbk || !read_range_cbk || !get_last_error_cbk) { + return false; + } + ctx->networking.open = open_cbk; + ctx->networking.close = close_cbk; + ctx->networking.get_header_value = get_header_value_cbk; + ctx->networking.get_file_size = get_file_size_cbk; + ctx->networking.read_range = read_range_cbk; + ctx->networking.get_last_error = get_last_error_cbk; + ctx->networking.user_data = user_data; + return true; +} diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 19478c48..0e830450 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -35,6 +35,8 @@ NS_PROJ_START +//! @cond Doxygen_Suppress + class File; class FileManager { @@ -48,6 +50,8 @@ class FileManager { // "High-level" interface, honoring PROJ_LIB and the like. static std::unique_ptr open_resource_file(PJ_CONTEXT *ctx, const char *name); + + static void fillDefaultNetworkInterface(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -63,6 +67,8 @@ class File { virtual unsigned long long tell() = 0; }; +//! @endcond Doxygen_Suppress + NS_PROJ_END #endif // FILEMANAGER_HPP_INCLUDED \ No newline at end of file diff --git a/src/open_lib.cpp b/src/open_lib.cpp index e1572754..6bdd5510 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -53,6 +53,8 @@ PROJ_LIB; nullptr; #endif +using namespace NS_PROJ::internal; + /************************************************************************/ /* pj_set_finder() */ /************************************************************************/ @@ -238,7 +240,9 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, else if (strchr(dir_chars,*name) || (*name == '.' && strchr(dir_chars,name[1])) || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) - || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) ) + || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) + || starts_with(name, "http://") + || starts_with(name, "https://")) sysname = name; /* or try to use application provided file finder */ diff --git a/src/proj.h b/src/proj.h index fc309542..ba6e7043 100644 --- a/src/proj.h +++ b/src/proj.h @@ -353,6 +353,82 @@ void PROJ_DLL proj_context_set_search_paths(PJ_CONTEXT *ctx, int count_paths, co void PROJ_DLL proj_context_use_proj4_init_rules(PJ_CONTEXT *ctx, int enable); int PROJ_DLL proj_context_get_use_proj4_init_rules(PJ_CONTEXT *ctx, int from_legacy_code_path); +/*! @endcond */ + +/** Opaque structure for PROJ. Implementations might cast it to their + * structure/class of choice. */ +typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; + +/** Network access: open callback + * + * Should try to read the size_to_read first bytes of the file of URL url, + * and write them to buffer. *out_size_read should be updated with the actual + * amount of bytes read (== size_to_read if the file is larger than size_to_read). + * During this read, the implementation should make sure to store the HTTP + * headers from the server response to be able to respond to + * proj_network_get_header_value_cbk_type and proj_network_get_file_size_cbk_type + * callbacks. + * + * @return a non-NULL opaque handle in case of success. + */ +typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( + PJ_CONTEXT* ctx, + const char* url, + size_t size_to_read, + void* buffer, + size_t* out_size_read, + void* user_data); + +/** Network access: close callback */ +typedef void (*proj_network_close_cbk_type)(PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + void* user_data); + +/** Network access: get HTTP headers */ +typedef const char* (*proj_network_get_header_value_cbk_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + const char* header_name, + void* user_data); + +/** Network access: get file size */ +typedef unsigned long long (*proj_network_get_file_size_cbk_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + void* user_data); + +/** Network access: read range + * + * Read size_to_read bytes from handle, starting at offset, into + * buffer. + * @return the number of bytes actually read (0 in case of error) + */ +typedef size_t (*proj_network_read_range_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + unsigned long long offset, + size_t size_to_read, + void* buffer, + void* user_data); + +/** Network access: get last error message */ +typedef const char* (*proj_network_get_last_error_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE*, + void* user_data); + +int PROJ_DLL proj_context_set_network_callbacks( + PJ_CONTEXT* ctx, + proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_get_file_size_cbk_type get_file_size_cbk, + proj_network_read_range_type read_range_cbk, + proj_network_get_last_error_type get_last_error_cbk, + void* user_data); + +/*! @cond Doxygen_Suppress */ + /* Manage the transformation definition object PJ */ PJ PROJ_DLL *proj_create (PJ_CONTEXT *ctx, const char *definition); PJ PROJ_DLL *proj_create_argv (PJ_CONTEXT *ctx, int argc, char **argv); diff --git a/src/proj_internal.h b/src/proj_internal.h index 93134946..aefeea39 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -666,6 +666,17 @@ struct projFileAPI_t; struct projCppContext; +struct projNetworkCallbacksAndData +{ + proj_network_open_cbk_type open = nullptr; + proj_network_close_cbk_type close = nullptr; + proj_network_get_header_value_cbk_type get_header_value = nullptr; + proj_network_get_file_size_cbk_type get_file_size = nullptr; + proj_network_read_range_type read_range = nullptr; + proj_network_get_last_error_type get_last_error = nullptr; + void* user_data = nullptr; +}; + /* proj thread context */ struct projCtx_t { int last_errno = 0; @@ -684,6 +695,8 @@ struct projCtx_t { const char* (*file_finder) (PJ_CONTEXT *, const char*, void* user_data) = nullptr; void* file_finder_user_data = nullptr; + projNetworkCallbacksAndData networking{}; + int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() projCtx_t() = default; -- cgit v1.2.3 From f73527fa20c9250cfced6aa73a7e46f40dc2c214 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 22:36:07 +0100 Subject: Add very minimalistic and slow libcurl implementation --- src/filemanager.cpp | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 97f6a4dc..3a2acee4 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -29,11 +29,18 @@ #define FROM_PROJ_CPP #endif +#include + #include "filemanager.hpp" #include "proj.h" #include "proj/internal/internal.hpp" #include "proj_internal.h" +#ifdef CURL_ENABLED +#include +#include // for sqlite3_snprintf +#endif + //! @cond Doxygen_Suppress NS_PROJ_START @@ -258,6 +265,152 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { // --------------------------------------------------------------------------- +#ifdef CURL_ENABLED + +struct CurlFileHandle { + CURL *m_handle = nullptr; + std::string m_headers; + + CurlFileHandle(const CurlFileHandle &) = delete; + CurlFileHandle &operator=(const CurlFileHandle &) = delete; + + explicit CurlFileHandle(CURL *handle, std::string &&headers) + : m_handle(handle), m_headers(std::move(headers)) {} + ~CurlFileHandle(); +}; + +// --------------------------------------------------------------------------- + +CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } + +// --------------------------------------------------------------------------- + +static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, + void *req) { + const size_t nSize = count * nmemb; + auto pStr = static_cast(req); + if (pStr->size() + nSize > pStr->capacity()) { + // to avoid servers not honouring Range to cause excessive memory + // allocation + return 0; + } + pStr->append(static_cast(buffer), nSize); + return nmemb; +} + +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, + size_t size_to_read, void *buffer, + size_t *out_size_read, void *) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return nullptr; + curl_easy_setopt(hCurlHandle, CURLOPT_URL, url); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(hCurlHandle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(hCurlHandle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(hCurlHandle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "0-%llu", size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string headers; + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, pj_curl_write_func); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, nullptr); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + curl_easy_cleanup(hCurlHandle); + return nullptr; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + *out_size_read = std::min(size_to_read, body.size()); + + return reinterpret_cast( + new CurlFileHandle(hCurlHandle, std::move(headers))); +} + +// --------------------------------------------------------------------------- + +static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, + void * /*user_data*/) { + delete reinterpret_cast(handle); +} + +// --------------------------------------------------------------------------- + +static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, + unsigned long long offset, size_t size_to_read, + void *buffer, void *) { + auto handle = reinterpret_cast(raw_handle); + auto hCurlHandle = handle->m_handle; + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + return 0; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + return std::min(size_to_read, body.size()); +} + +#else + +// --------------------------------------------------------------------------- + static PROJ_NETWORK_HANDLE * no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, size_t, /* size to read */ @@ -281,12 +434,20 @@ static const char *no_op_network_get_last_error(PJ_CONTEXT *, return "Network functionality not available"; } +#endif + // --------------------------------------------------------------------------- void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { +#ifdef CURL_ENABLED + ctx->networking.open = pj_curl_open; + ctx->networking.close = pj_curl_close; + ctx->networking.read_range = pj_curl_read_range; +#else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; ctx->networking.get_last_error = no_op_network_get_last_error; +#endif } // --------------------------------------------------------------------------- -- cgit v1.2.3 From ed92965937856ae697010e1461e82f6245d19909 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 20 Dec 2019 19:10:20 +0100 Subject: Network: add a memory cache and I/O chunking strategy --- src/filemanager.cpp | 259 ++++++++++++++++++++++++++++++++++++++++++++++++---- src/filemanager.hpp | 2 + src/malloc.cpp | 2 + src/proj.h | 16 ++-- 4 files changed, 256 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 3a2acee4..2dfcd6b6 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -28,14 +28,34 @@ #ifndef FROM_PROJ_CPP #define FROM_PROJ_CPP #endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include +#include +#include #include "filemanager.hpp" #include "proj.h" #include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" #include "proj_internal.h" +#ifdef __MINGW32__ +// mingw32-win32 doesn't implement std::mutex +namespace { +class MyMutex { + public: + // cppcheck-suppress functionStatic + void lock() { pj_acquire_lock(); } + // cppcheck-suppress functionStatic + void unlock() { pj_release_lock(); } +}; +} +#else +#include +#define MyMutex std::mutex +#endif + #ifdef CURL_ENABLED #include #include // for sqlite3_snprintf @@ -177,17 +197,97 @@ std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +constexpr size_t DOWNLOAD_CHUNK_SIZE = 16 * 1024; +constexpr int MAX_CHUNKS = 64; + +class NetworkChunkCache { + public: + void insert(const std::string &url, unsigned long long chunkIdx, + std::vector &&data); + + std::shared_ptr> + get(const std::string &url, unsigned long long chunkIdx); + + void clear(); + + private: + struct Key { + std::string url; + unsigned long long chunkIdx; + + Key(const std::string &urlIn, unsigned long long chunkIdxIn) + : url(urlIn), chunkIdx(chunkIdxIn) {} + bool operator==(const Key &other) const { + return url == other.url && chunkIdx == other.chunkIdx; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return std::hash{}(k.url) ^ + (std::hash{}(k.chunkIdx) << 1); + } + }; + + lru11::Cache< + Key, std::shared_ptr>, MyMutex, + std::unordered_map< + Key, + typename std::list>>>::iterator, + KeyHasher>> + cache_{MAX_CHUNKS}; +}; + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::insert(const std::string &url, + unsigned long long chunkIdx, + std::vector &&data) { + cache_.insert( + Key(url, chunkIdx), + std::make_shared>(std::move(data))); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr> +NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) { + std::shared_ptr> ret; + cache_.tryGet(Key(url, chunkIdx), ret); + return ret; +} + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::clear() +{ + cache_.clear(); +} + +// --------------------------------------------------------------------------- + +static NetworkChunkCache gNetworkChunkCache{}; + +// --------------------------------------------------------------------------- + class NetworkFile : public File { PJ_CONTEXT *m_ctx; + std::string m_url; PROJ_NETWORK_HANDLE *m_handle; unsigned long long m_pos = 0; + size_t m_nBlocksToDownload = 1; + unsigned long long m_lastDownloadedOffset; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; protected: - NetworkFile(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle) - : m_ctx(ctx), m_handle(handle) {} + NetworkFile(PJ_CONTEXT *ctx, const std::string &url, + PROJ_NETWORK_HANDLE *handle, + unsigned long long lastDownloadOffset) + : m_ctx(ctx), m_url(url), m_handle(handle), + m_lastDownloadedOffset(lastDownloadOffset) {} public: ~NetworkFile() override; @@ -202,20 +302,125 @@ class NetworkFile : public File { // --------------------------------------------------------------------------- std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { - std::vector buffer(16 * 1024); - size_t size_read = 0; - auto handle = ctx->networking.open(ctx, filename, buffer.size(), &buffer[0], - &size_read, ctx->networking.user_data); - return std::unique_ptr(handle ? new NetworkFile(ctx, handle) - : nullptr); + if (gNetworkChunkCache.get(filename, 0)) { + return std::unique_ptr( + new NetworkFile(ctx, filename, nullptr, + std::numeric_limits::max())); + } else { + std::vector buffer(DOWNLOAD_CHUNK_SIZE); + size_t size_read = 0; + auto handle = + ctx->networking.open(ctx, filename, 0, buffer.size(), &buffer[0], + &size_read, ctx->networking.user_data); + buffer.resize(size_read); + gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + return std::unique_ptr( + handle ? new NetworkFile(ctx, filename, handle, size_read) + : nullptr); + } } // --------------------------------------------------------------------------- size_t NetworkFile::read(void *buffer, size_t sizeBytes) { - size_t nRead = m_ctx->networking.read_range( - m_ctx, m_handle, m_pos, sizeBytes, buffer, m_ctx->networking.user_data); - m_pos += nRead; + + if (sizeBytes == 0) + return 0; + + auto iterOffset = m_pos; + while (sizeBytes) { + const auto chunkIdxToDownload = iterOffset / DOWNLOAD_CHUNK_SIZE; + const auto offsetToDownload = chunkIdxToDownload * DOWNLOAD_CHUNK_SIZE; + std::vector region; + auto pChunk = gNetworkChunkCache.get(m_url, chunkIdxToDownload); + if (pChunk != nullptr) { + region = *pChunk; + } else { + if (offsetToDownload == m_lastDownloadedOffset) { + // In case of consecutive reads (of small size), we use a + // heuristic that we will read the file sequentially, so + // we double the requested size to decrease the number of + // client/server roundtrips. + if (m_nBlocksToDownload < 100) + m_nBlocksToDownload *= 2; + } else { + // Random reads. Cancel the above heuristics. + m_nBlocksToDownload = 1; + } + + // Ensure that we will request at least the number of blocks + // to satisfy the remaining buffer size to read. + const auto endOffsetToDownload = + ((iterOffset + sizeBytes + DOWNLOAD_CHUNK_SIZE - 1) / + DOWNLOAD_CHUNK_SIZE) * + DOWNLOAD_CHUNK_SIZE; + const auto nMinBlocksToDownload = static_cast( + (endOffsetToDownload - offsetToDownload) / DOWNLOAD_CHUNK_SIZE); + if (m_nBlocksToDownload < nMinBlocksToDownload) + m_nBlocksToDownload = nMinBlocksToDownload; + + // Avoid reading already cached data. + // Note: this might get evicted if concurrent reads are done, but + // this should not cause bugs. Just missed optimization. + for (size_t i = 1; i < m_nBlocksToDownload; i++) { + if (gNetworkChunkCache.get(m_url, chunkIdxToDownload + i) != + nullptr) { + m_nBlocksToDownload = i; + break; + } + } + + if (m_nBlocksToDownload > MAX_CHUNKS) + m_nBlocksToDownload = MAX_CHUNKS; + + region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); + size_t nRead = 0; + if (!m_handle) { + m_handle = m_ctx->networking.open( + m_ctx, m_url.c_str(), offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + &nRead, m_ctx->networking.user_data); + if (!m_handle) { + return 0; + } + } else { + nRead = m_ctx->networking.read_range( + m_ctx, m_handle, offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + m_ctx->networking.user_data); + } + if (nRead == 0) { + return 0; + } + region.resize(nRead); + m_lastDownloadedOffset = offsetToDownload + nRead; + + const auto nChunks = + (region.size() + DOWNLOAD_CHUNK_SIZE - 1) / DOWNLOAD_CHUNK_SIZE; + for (size_t i = 0; i < nChunks; i++) { + std::vector chunk( + region.data() + i * DOWNLOAD_CHUNK_SIZE, + region.data() + + std::min((i + 1) * DOWNLOAD_CHUNK_SIZE, region.size())); + gNetworkChunkCache.insert(m_url, chunkIdxToDownload + i, + std::move(chunk)); + } + } + const size_t nToCopy = static_cast( + std::min(static_cast(sizeBytes), + region.size() - (iterOffset - offsetToDownload))); + memcpy(buffer, region.data() + iterOffset - offsetToDownload, nToCopy); + buffer = static_cast(buffer) + nToCopy; + iterOffset += nToCopy; + sizeBytes -= nToCopy; + if (region.size() < static_cast(DOWNLOAD_CHUNK_SIZE) && + sizeBytes != 0) { + break; + } + } + + size_t nRead = static_cast(iterOffset - m_pos); + m_pos = iterOffset; return nRead; } @@ -229,6 +434,16 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; + if (!m_handle) { + size_t nRead = 0; + char dummy; + m_handle = + m_ctx->networking.open(m_ctx, m_url.c_str(), 0, 1, &dummy, + &nRead, m_ctx->networking.user_data); + if (!m_handle) { + return false; + } + } const auto filesize = m_ctx->networking.get_file_size( m_ctx, m_handle, m_ctx->networking.user_data); if (filesize == 0) @@ -245,7 +460,9 @@ unsigned long long NetworkFile::tell() { return m_pos; } // --------------------------------------------------------------------------- NetworkFile::~NetworkFile() { - m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + if (m_handle) { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + } } // --------------------------------------------------------------------------- @@ -301,6 +518,7 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, + unsigned long long offset, size_t size_to_read, void *buffer, size_t *out_size_read, void *) { CURL *hCurlHandle = curl_easy_init(); @@ -326,7 +544,8 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, } char szBuffer[128]; - sqlite3_snprintf(sizeof(szBuffer), szBuffer, "0-%llu", size_to_read - 1); + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); std::string headers; @@ -413,9 +632,10 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, static PROJ_NETWORK_HANDLE * no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, - size_t, /* size to read */ - void *, /* buffer to update with bytes read*/ - size_t *, /* output: size actually read */ + unsigned long long, /* offset */ + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ void * /*user_data*/) { pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); return nullptr; @@ -452,6 +672,13 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- +void FileManager::clearCache() +{ + gNetworkChunkCache.clear(); +} + +// --------------------------------------------------------------------------- + NS_PROJ_END //! @endcond diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 0e830450..949b223f 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -52,6 +52,8 @@ class FileManager { const char *name); static void fillDefaultNetworkInterface(PJ_CONTEXT *ctx); + + static void clearCache(); }; // --------------------------------------------------------------------------- diff --git a/src/malloc.cpp b/src/malloc.cpp index 3fd3699f..1c539b6b 100644 --- a/src/malloc.cpp +++ b/src/malloc.cpp @@ -49,6 +49,7 @@ #include "proj.h" #include "proj_internal.h" #include "grids.hpp" +#include "filemanager.hpp" using namespace NS_PROJ; @@ -262,4 +263,5 @@ void proj_cleanup() { /*****************************************************************************/ pj_clear_initcache(); pj_deallocate_grids(); + FileManager::clearCache(); } diff --git a/src/proj.h b/src/proj.h index ba6e7043..5f080285 100644 --- a/src/proj.h +++ b/src/proj.h @@ -361,7 +361,8 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; /** Network access: open callback * - * Should try to read the size_to_read first bytes of the file of URL url, + * Should try to read the size_to_read first bytes at the specified offset of + * the file given by URL url, * and write them to buffer. *out_size_read should be updated with the actual * amount of bytes read (== size_to_read if the file is larger than size_to_read). * During this read, the implementation should make sure to store the HTTP @@ -372,12 +373,13 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; * @return a non-NULL opaque handle in case of success. */ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( - PJ_CONTEXT* ctx, - const char* url, - size_t size_to_read, - void* buffer, - size_t* out_size_read, - void* user_data); + PJ_CONTEXT* ctx, + const char* url, + unsigned long long offset, + size_t size_to_read, + void* buffer, + size_t* out_size_read, + void* user_data); /** Network access: close callback */ typedef void (*proj_network_close_cbk_type)(PJ_CONTEXT* ctx, -- cgit v1.2.3 From d3bdbb841719780560438129b4911a86a7a4be58 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 21 Dec 2019 00:38:50 +0100 Subject: Add testing of network functionality --- src/filemanager.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 2dfcd6b6..cc692616 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -260,10 +260,7 @@ NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) { // --------------------------------------------------------------------------- -void NetworkChunkCache::clear() -{ - cache_.clear(); -} +void NetworkChunkCache::clear() { cache_.clear(); } // --------------------------------------------------------------------------- @@ -672,10 +669,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -void FileManager::clearCache() -{ - gNetworkChunkCache.clear(); -} +void FileManager::clearCache() { gNetworkChunkCache.clear(); } // --------------------------------------------------------------------------- -- cgit v1.2.3 From 9d0bd793b552e248a10f9ff8b6c62d942fe437a1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Dec 2019 15:58:47 +0100 Subject: Network: remove dedicated get_file_size() callback to use get_header_value(), and test --- src/filemanager.cpp | 93 +++++++++++++++++++++++++++++++++++++---------------- src/proj.h | 10 +----- src/proj_internal.h | 1 - 3 files changed, 67 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index cc692616..ea0a63ea 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -266,6 +266,13 @@ void NetworkChunkCache::clear() { cache_.clear(); } static NetworkChunkCache gNetworkChunkCache{}; +struct FileProperties { + unsigned long long size; +}; + +static lru11::Cache + gNetworkFileProperties{}; + // --------------------------------------------------------------------------- class NetworkFile : public File { @@ -275,6 +282,7 @@ class NetworkFile : public File { unsigned long long m_pos = 0; size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; + unsigned long long m_filesize; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; @@ -282,9 +290,10 @@ class NetworkFile : public File { protected: NetworkFile(PJ_CONTEXT *ctx, const std::string &url, PROJ_NETWORK_HANDLE *handle, - unsigned long long lastDownloadOffset) + unsigned long long lastDownloadOffset, + unsigned long long filesize) : m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset) {} + m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} public: ~NetworkFile() override; @@ -300,9 +309,14 @@ class NetworkFile : public File { std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { if (gNetworkChunkCache.get(filename, 0)) { - return std::unique_ptr( - new NetworkFile(ctx, filename, nullptr, - std::numeric_limits::max())); + unsigned long long filesize = 0; + FileProperties props; + if (gNetworkFileProperties.tryGet(filename, props)) { + filesize = props.size; + } + return std::unique_ptr(new NetworkFile( + ctx, filename, nullptr, + std::numeric_limits::max(), filesize)); } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; @@ -311,8 +325,24 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { &size_read, ctx->networking.user_data); buffer.resize(size_read); gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + + unsigned long long filesize = 0; + if (handle) { + const char *contentRange = ctx->networking.get_header_value( + ctx, handle, "Content-Range", ctx->networking.user_data); + if (contentRange) { + const char *slash = strchr(contentRange, '/'); + if (slash) { + filesize = std::stoull(slash + 1); + FileProperties props; + props.size = filesize; + gNetworkFileProperties.insert(filename, props); + } + } + } + return std::unique_ptr( - handle ? new NetworkFile(ctx, filename, handle, size_read) + handle ? new NetworkFile(ctx, filename, handle, size_read, filesize) : nullptr); } } @@ -431,21 +461,7 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; - if (!m_handle) { - size_t nRead = 0; - char dummy; - m_handle = - m_ctx->networking.open(m_ctx, m_url.c_str(), 0, 1, &dummy, - &nRead, m_ctx->networking.user_data); - if (!m_handle) { - return false; - } - } - const auto filesize = m_ctx->networking.get_file_size( - m_ctx, m_handle, m_ctx->networking.user_data); - if (filesize == 0) - return false; - m_pos = filesize; + m_pos = m_filesize; } return true; } @@ -484,6 +500,7 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { struct CurlFileHandle { CURL *m_handle = nullptr; std::string m_headers; + std::string m_lastval{}; CurlFileHandle(const CurlFileHandle &) = delete; CurlFileHandle &operator=(const CurlFileHandle &) = delete; @@ -623,6 +640,27 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, return std::min(size_to_read, body.size()); } +// --------------------------------------------------------------------------- + +static const char *pj_curl_get_header_value(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *raw_handle, + const char *header_name, void *) { + auto handle = reinterpret_cast(raw_handle); + auto pos = ci_find(handle->m_headers, header_name); + if (pos == std::string::npos) + return nullptr; + const char *c_str = handle->m_headers.c_str(); + if (c_str[pos] == ':') + pos++; + while (c_str[pos] == ' ') + pos++; + auto posEnd = pos; + while (c_str[posEnd] != '\n' && c_str[posEnd] != '\0') + posEnd++; + handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos); + return handle->m_lastval.c_str(); +} + #else // --------------------------------------------------------------------------- @@ -660,6 +698,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { ctx->networking.open = pj_curl_open; ctx->networking.close = pj_curl_close; ctx->networking.read_range = pj_curl_read_range; + ctx->networking.get_header_value = pj_curl_get_header_value; #else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; @@ -669,7 +708,10 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -void FileManager::clearCache() { gNetworkChunkCache.clear(); } +void FileManager::clearCache() { + gNetworkChunkCache.clear(); + gNetworkFileProperties.clear(); +} // --------------------------------------------------------------------------- @@ -687,7 +729,6 @@ NS_PROJ_END * @param open_cbk Callback to open a remote file given its URL * @param close_cbk Callback to close a remote file. * @param get_header_value_cbk Callback to get HTTP headers - * @param get_file_size_cbk Callback to get the size of the remote file. * @param read_range_cbk Callback to read a range of bytes inside a remote file. * @param get_last_error_cbk Callback to get last error message. * @param user_data Arbitrary pointer provided by the user, and passed to the @@ -698,20 +739,18 @@ int proj_context_set_network_callbacks( PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_get_file_size_cbk_type get_file_size_cbk, proj_network_read_range_type read_range_cbk, proj_network_get_last_error_type get_last_error_cbk, void *user_data) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } - if (!open_cbk || !close_cbk || !get_header_value_cbk || - !get_file_size_cbk || !read_range_cbk || !get_last_error_cbk) { + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk || + !get_last_error_cbk) { return false; } ctx->networking.open = open_cbk; ctx->networking.close = close_cbk; ctx->networking.get_header_value = get_header_value_cbk; - ctx->networking.get_file_size = get_file_size_cbk; ctx->networking.read_range = read_range_cbk; ctx->networking.get_last_error = get_last_error_cbk; ctx->networking.user_data = user_data; diff --git a/src/proj.h b/src/proj.h index 5f080285..95bbacd1 100644 --- a/src/proj.h +++ b/src/proj.h @@ -367,8 +367,7 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; * amount of bytes read (== size_to_read if the file is larger than size_to_read). * During this read, the implementation should make sure to store the HTTP * headers from the server response to be able to respond to - * proj_network_get_header_value_cbk_type and proj_network_get_file_size_cbk_type - * callbacks. + * proj_network_get_header_value_cbk_type callback. * * @return a non-NULL opaque handle in case of success. */ @@ -393,12 +392,6 @@ typedef const char* (*proj_network_get_header_value_cbk_type)( const char* header_name, void* user_data); -/** Network access: get file size */ -typedef unsigned long long (*proj_network_get_file_size_cbk_type)( - PJ_CONTEXT* ctx, - PROJ_NETWORK_HANDLE* handle, - void* user_data); - /** Network access: read range * * Read size_to_read bytes from handle, starting at offset, into @@ -424,7 +417,6 @@ int PROJ_DLL proj_context_set_network_callbacks( proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_get_file_size_cbk_type get_file_size_cbk, proj_network_read_range_type read_range_cbk, proj_network_get_last_error_type get_last_error_cbk, void* user_data); diff --git a/src/proj_internal.h b/src/proj_internal.h index aefeea39..0c148d02 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -671,7 +671,6 @@ struct projNetworkCallbacksAndData proj_network_open_cbk_type open = nullptr; proj_network_close_cbk_type close = nullptr; proj_network_get_header_value_cbk_type get_header_value = nullptr; - proj_network_get_file_size_cbk_type get_file_size = nullptr; proj_network_read_range_type read_range = nullptr; proj_network_get_last_error_type get_last_error = nullptr; void* user_data = nullptr; -- cgit v1.2.3 From 0a1f1fe469029ae31591dc8b51d20f5617496128 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Dec 2019 23:12:26 +0100 Subject: Network: only enable it if PROJ_NETWORK=ON or proj_context_set_enable_network(ctx, true) --- src/filemanager.cpp | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++-- src/proj.h | 3 +++ src/proj_internal.h | 4 ++++ 3 files changed, 63 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index ea0a63ea..cd738d5e 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -63,9 +63,9 @@ class MyMutex { //! @cond Doxygen_Suppress -NS_PROJ_START +using namespace NS_PROJ::internal; -using namespace internal; +NS_PROJ_START // --------------------------------------------------------------------------- @@ -488,6 +488,14 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { } #endif if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + if (!pj_context_is_network_enabled(ctx)) { + pj_log( + ctx, PJ_LOG_ERROR, + "Attempt at accessing remote resource not authorized. Either " + "set PROJ_NETWORK=ON or " + "proj_context_set_enable_network(ctx, TRUE)"); + return nullptr; + } return NetworkFile::open(ctx, filename); } return FileStdio::open(ctx, filename); @@ -756,3 +764,49 @@ int proj_context_set_network_callbacks( ctx->networking.user_data = user_data; return true; } + +// --------------------------------------------------------------------------- + +/** Enable or disable network access. +* +* @param ctx PROJ context, or NULL +* @param enable TRUE if network access is allowed. +* @return TRUE if network access is possible. That is either libcurl is +* available, or an alternate interface has been set. +*/ +int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + ctx->networking.enabled_env_variable_checked = true; + ctx->networking.enabled = enable != FALSE; +#ifdef CURL_ENABLED + return ctx->networking.enabled; +#else + return ctx->networking.enabled && + ctx->networking.open != NS_PROJ::no_op_network_open; +#endif +} + +// --------------------------------------------------------------------------- + +//! @cond Doxygen_Suppress + +bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (ctx->networking.enabled_env_variable_checked) { + return ctx->networking.enabled; + } + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled && enabled[0] != '\0') { + ctx->networking.enabled = ci_equal(enabled, "ON") || + ci_equal(enabled, "YES") || + ci_equal(enabled, "TRUE"); + } + ctx->networking.enabled_env_variable_checked = true; + return ctx->networking.enabled; +} + +//! @endcond diff --git a/src/proj.h b/src/proj.h index 95bbacd1..4f7878a4 100644 --- a/src/proj.h +++ b/src/proj.h @@ -421,6 +421,9 @@ int PROJ_DLL proj_context_set_network_callbacks( proj_network_get_last_error_type get_last_error_cbk, void* user_data); +int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, + int enabled); + /*! @cond Doxygen_Suppress */ /* Manage the transformation definition object PJ */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 0c148d02..d54d8fb9 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -668,6 +668,8 @@ struct projCppContext; struct projNetworkCallbacksAndData { + bool enabled = false; + bool enabled_env_variable_checked = false; // whereas we have checked PROJ_NETWORK env variable proj_network_open_cbk_type open = nullptr; proj_network_close_cbk_type close = nullptr; proj_network_get_header_value_cbk_type get_header_value = nullptr; @@ -824,6 +826,8 @@ PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); +bool pj_context_is_network_enabled(PJ_CONTEXT* ctx); + /* classic public API */ #include "proj_api.h" -- cgit v1.2.3 From c4589fbe42e5fea07a03919d3484164f5fb70dd3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 18:44:45 +0100 Subject: Network: automatically use CDN resources when local resources not available, and networking enabled --- src/4D_api.cpp | 7 +- src/apps/projinfo.cpp | 30 +++++--- src/filemanager.cpp | 15 ++-- src/filemanager.hpp | 5 +- src/grids.cpp | 26 ++++--- src/iso19111/c_api.cpp | 20 +++++- src/iso19111/coordinateoperation.cpp | 131 ++++++++++++++++++++++++----------- src/iso19111/factory.cpp | 77 ++++++++++++-------- src/open_lib.cpp | 59 ++++++++++++---- src/proj.h | 6 ++ src/proj_internal.h | 4 +- src/transformations/hgridshift.cpp | 40 ++++++++--- src/transformations/vgridshift.cpp | 65 +++++++++++++++-- 13 files changed, 355 insertions(+), 130 deletions(-) (limited to 'src') diff --git a/src/4D_api.cpp b/src/4D_api.cpp index efb4a86a..cee8262e 100644 --- a/src/4D_api.cpp +++ b/src/4D_api.cpp @@ -262,7 +262,7 @@ similarly, but prefers the 2D resp. 3D interfaces if available. auto coordOperation = dynamic_cast< NS_PROJ::operation::CoordinateOperation*>(alt.pj->iso_obj.get()); if( coordOperation ) { - if( coordOperation->gridsNeeded(dbContext).empty() ) { + if( coordOperation->gridsNeeded(dbContext, true).empty() ) { if( P->iCurCoordOp != i ) { std::string msg("Using coordinate operation "); msg += alt.name; @@ -1117,7 +1117,10 @@ PJ *proj_create_crs_to_crs_from_pj (PJ_CONTEXT *ctx, const PJ *source_crs, cons proj_operation_factory_context_set_spatial_criterion( ctx, operation_ctx, PROJ_SPATIAL_CRITERION_PARTIAL_INTERSECTION); proj_operation_factory_context_set_grid_availability_use( - ctx, operation_ctx, PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); + ctx, operation_ctx, + pj_context_is_network_enabled(ctx) ? + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); auto op_list = proj_create_operations(ctx, source_crs, target_crs, operation_ctx); diff --git a/src/apps/projinfo.cpp b/src/apps/projinfo.cpp index fd9b2f46..f13e526b 100644 --- a/src/apps/projinfo.cpp +++ b/src/apps/projinfo.cpp @@ -87,7 +87,8 @@ static void usage() { << " [--spatial-test contains|intersects]" << std::endl << " [--crs-extent-use none|both|intersection|smallest]" << std::endl - << " [--grid-check none|discard_missing|sort] " + << " [--grid-check " + "none|discard_missing|sort|known_available] " "[--show-superseded]" << std::endl << " [--pivot-crs always|if_no_direct_transformation|" @@ -509,7 +510,7 @@ static void outputObject( auto op = dynamic_cast(obj.get()); if (op && dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); bool firstWarning = true; for (const auto &grid : setGrids) { if (!grid.available) { @@ -525,6 +526,7 @@ static void outputObject( if (!grid.url.empty()) { std::cout << " at " << grid.url; } + std::cout << ", or on CDN"; } else if (!grid.url.empty()) { std::cout << " Can be obtained at " << grid.url; } @@ -539,8 +541,9 @@ static void outputObject( // --------------------------------------------------------------------------- -static void outputOperationSummary(const CoordinateOperationNNPtr &op, - const DatabaseContextPtr &dbContext) { +static void outputOperationSummary( + const CoordinateOperationNNPtr &op, const DatabaseContextPtr &dbContext, + CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse) { auto ids = op->identifiers(); if (!ids.empty()) { std::cout << *(ids[0]->codeSpace()) << ":" << ids[0]->code(); @@ -586,10 +589,16 @@ static void outputOperationSummary(const CoordinateOperationNNPtr &op, if (dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); for (const auto &grid : setGrids) { if (!grid.available) { std::cout << ", at least one grid missing"; + if (gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE && + !grid.packageName.empty()) { + std::cout << " on the system, but available on CDN"; + } break; } } @@ -686,7 +695,7 @@ static void outputOperations( } if (summary) { for (const auto &op : list) { - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); } } else { bool first = true; @@ -701,7 +710,7 @@ static void outputOperations( "\xC2\xB0" << (i + 1) << ":" << std::endl << std::endl; - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); std::cout << std::endl; outputObject(dbContext, op, allowUseIntermediateCRS, outputOpt); } @@ -734,7 +743,9 @@ int main(int argc, char **argv) { CoordinateOperationContext::SourceTargetCRSExtentUse::SMALLEST; bool buildBoundCRSToWGS84 = false; CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse = - CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; + pj_context_is_network_enabled(nullptr) + ? CoordinateOperationContext::GridAvailabilityUse::KNOWN_AVAILABLE + : CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; CoordinateOperationContext::IntermediateCRSUse allowUseIntermediateCRS = CoordinateOperationContext::IntermediateCRSUse:: IF_NO_DIRECT_TRANSFORMATION; @@ -947,6 +958,9 @@ int main(int argc, char **argv) { } else if (ci_equal(value, "sort")) { gridAvailabilityUse = CoordinateOperationContext:: GridAvailabilityUse::USE_FOR_SORTING; + } else if (ci_equal(value, "known_available")) { + gridAvailabilityUse = CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE; } else { std::cerr << "Unrecognized value for option --grid-check: " << value << std::endl; diff --git a/src/filemanager.cpp b/src/filemanager.cpp index cd738d5e..d9a02632 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -69,7 +69,7 @@ NS_PROJ_START // --------------------------------------------------------------------------- -File::File() = default; +File::File(const std::string &name) : name_(name) {} // --------------------------------------------------------------------------- @@ -85,7 +85,8 @@ class FileStdio : public File { FileStdio &operator=(const FileStdio &) = delete; protected: - FileStdio(PJ_CONTEXT *ctx, FILE *fp) : m_ctx(ctx), m_fp(fp) {} + FileStdio(const std::string &name, PJ_CONTEXT *ctx, FILE *fp) + : File(name), m_ctx(ctx), m_fp(fp) {} public: ~FileStdio() override; @@ -130,7 +131,8 @@ unsigned long long FileStdio::tell() { std::unique_ptr FileStdio::open(PJ_CONTEXT *ctx, const char *filename) { auto fp = fopen(filename, "rb"); - return std::unique_ptr(fp ? new FileStdio(ctx, fp) : nullptr); + return std::unique_ptr(fp ? new FileStdio(filename, ctx, fp) + : nullptr); } // --------------------------------------------------------------------------- @@ -145,7 +147,8 @@ class FileLegacyAdapter : public File { FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; protected: - FileLegacyAdapter(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + FileLegacyAdapter(const std::string &name, PJ_CONTEXT *ctx, PAFile fp) + : File(name), m_ctx(ctx), m_fp(fp) {} public: ~FileLegacyAdapter() override; @@ -189,7 +192,7 @@ unsigned long long FileLegacyAdapter::tell() { std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, const char *filename) { auto fid = pj_ctx_fopen(ctx, filename, "rb"); - return std::unique_ptr(fid ? new FileLegacyAdapter(ctx, fid) + return std::unique_ptr(fid ? new FileLegacyAdapter(filename, ctx, fid) : nullptr); } @@ -292,7 +295,7 @@ class NetworkFile : public File { PROJ_NETWORK_HANDLE *handle, unsigned long long lastDownloadOffset, unsigned long long filesize) - : m_ctx(ctx), m_url(url), m_handle(handle), + : File(url), m_ctx(ctx), m_url(url), m_handle(handle), m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} public: diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 949b223f..46391597 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -60,13 +60,16 @@ class FileManager { class File { protected: - File(); + std::string name_; + explicit File(const std::string &name); public: virtual ~File(); virtual size_t read(void *buffer, size_t sizeBytes) = 0; virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; virtual unsigned long long tell() = 0; + + const std::string &name() const { return name_; } }; //! @endcond Doxygen_Suppress diff --git a/src/grids.cpp b/src/grids.cpp index 91e51016..45e7e8b0 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -1299,14 +1299,15 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } - if (ends_with(filename, "gtx") || ends_with(filename, "GTX")) { - auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), filename); + const auto actualName(fp->name()); + if (ends_with(actualName, "gtx") || ends_with(actualName, "GTX")) { + auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), actualName); if (!grid) { return nullptr; } auto set = std::unique_ptr(new VerticalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "gtx"; set->m_grids.push_back(std::unique_ptr(grid)); return set; @@ -1324,7 +1325,7 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), filename); + auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), actualName); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; @@ -2130,6 +2131,7 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } + const auto actualName(fp->name()); char header[160]; /* -------------------------------------------------------------------- */ @@ -2151,35 +2153,35 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (header_size >= 144 + 16 && strncmp(header + 0, "HEADER", 6) == 0 && strncmp(header + 96, "W GRID", 6) == 0 && strncmp(header + 144, "TO NAD83 ", 16) == 0) { - auto grid = NTv1Grid::open(ctx, std::move(fp), filename); + auto grid = NTv1Grid::open(ctx, std::move(fp), actualName); if (!grid) { return nullptr; } auto set = std::unique_ptr( new HorizontalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "ntv1"; set->m_grids.push_back(std::unique_ptr(grid)); return set; } else if (header_size >= 9 && strncmp(header + 0, "CTABLE V2", 9) == 0) { - auto grid = CTable2Grid::open(ctx, std::move(fp), filename); + auto grid = CTable2Grid::open(ctx, std::move(fp), actualName); if (!grid) { return nullptr; } auto set = std::unique_ptr( new HorizontalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "ctable2"; set->m_grids.push_back(std::unique_ptr(grid)); return set; } else if (header_size >= 48 + 7 && strncmp(header + 0, "NUM_OREC", 8) == 0 && strncmp(header + 48, "GS_TYPE", 7) == 0) { - return NTv2GridSet::open(ctx, std::move(fp), filename); + return NTv2GridSet::open(ctx, std::move(fp), actualName); } else if (IsTIFF(header_size, reinterpret_cast(header))) { #ifdef TIFF_ENABLED - auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), filename); + auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), actualName); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; @@ -2450,6 +2452,7 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } + const auto actualName(fp->name()); /* -------------------------------------------------------------------- */ /* Load a header, to determine the file type. */ @@ -2463,7 +2466,8 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffGenericGridShiftSet::open(ctx, std::move(fp), filename); + auto set = + GTiffGenericGridShiftSet::open(ctx, std::move(fp), actualName); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; diff --git a/src/iso19111/c_api.cpp b/src/iso19111/c_api.cpp index 9db9e5b3..5df4c513 100644 --- a/src/iso19111/c_api.cpp +++ b/src/iso19111/c_api.cpp @@ -177,7 +177,11 @@ static PJ *pj_obj_create(PJ_CONTEXT *ctx, const IdentifiedObjectNNPtr &objIn) { auto formatter = PROJStringFormatter::create( PROJStringFormatter::Convention::PROJ_5, dbContext); auto projString = coordop->exportToPROJString(formatter.get()); + if (pj_context_is_network_enabled(ctx)) { + ctx->defer_grid_opening = true; + } auto pj = pj_create_internal(ctx, projString.c_str()); + ctx->defer_grid_opening = false; if (pj) { pj->iso_obj = objIn; if (ctx->cpp_context) { @@ -766,7 +770,7 @@ int PROJ_DLL proj_grid_get_info_from_database( bool open_license; bool available; if (!db_context->lookForGridInfo( - grid_name, ctx->cpp_context->lastGridFullName_, + grid_name, false, ctx->cpp_context->lastGridFullName_, ctx->cpp_context->lastGridPackageName_, ctx->cpp_context->lastGridUrl_, direct_download, open_license, available)) { @@ -6571,7 +6575,10 @@ int proj_coordoperation_is_instantiable(PJ_CONTEXT *ctx, } auto dbContext = getDBcontextNoException(ctx, __FUNCTION__); try { - auto ret = op->isPROJInstantiable(dbContext) ? 1 : 0; + auto ret = op->isPROJInstantiable(dbContext, + pj_context_is_network_enabled(ctx)) + ? 1 + : 0; if (ctx->cpp_context) { ctx->cpp_context->autoCloseDbIfNeeded(); } @@ -6883,7 +6890,8 @@ int proj_coordoperation_get_grid_used_count(PJ_CONTEXT *ctx, try { if (!coordoperation->gridsNeededAsked) { coordoperation->gridsNeededAsked = true; - const auto gridsNeeded = co->gridsNeeded(dbContext); + const auto gridsNeeded = + co->gridsNeeded(dbContext, pj_context_is_network_enabled(ctx)); for (const auto &gridDesc : gridsNeeded) { coordoperation->gridsNeeded.emplace_back(gridDesc); } @@ -7220,6 +7228,12 @@ void PROJ_DLL proj_operation_factory_context_set_grid_availability_use( CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY); break; + + case PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + factory_ctx->operationContext->setGridAvailabilityUse( + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); + break; } } catch (const std::exception &e) { proj_log_error(ctx, __FUNCTION__, e.what()); diff --git a/src/iso19111/coordinateoperation.cpp b/src/iso19111/coordinateoperation.cpp index 6120c768..7c0515c7 100644 --- a/src/iso19111/coordinateoperation.cpp +++ b/src/iso19111/coordinateoperation.cpp @@ -788,13 +788,15 @@ void CoordinateOperation::setAccuracies( * available. */ bool CoordinateOperation::isPROJInstantiable( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { try { exportToPROJString(io::PROJStringFormatter::create().get()); } catch (const std::exception &) { return false; } - for (const auto &gridDesc : gridsNeeded(databaseContext)) { + for (const auto &gridDesc : + gridsNeeded(databaseContext, considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return false; } @@ -2013,8 +2015,9 @@ bool SingleOperation::_isEquivalentTo(const util::IComparable *other, // --------------------------------------------------------------------------- -std::set SingleOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set +SingleOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set res; for (const auto &genOpParamvalue : parameterValues()) { auto opParamvalue = dynamic_cast( @@ -2026,9 +2029,9 @@ std::set SingleOperation::gridsNeeded( desc.shortName = value->valueFile(); if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, - desc.url, desc.directDownload, desc.openLicense, - desc.available); + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, + desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); } @@ -10209,10 +10212,12 @@ bool ConcatenatedOperation::_isEquivalentTo( // --------------------------------------------------------------------------- std::set ConcatenatedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set res; for (const auto &operation : operations()) { - const auto l_gridsNeeded = operation->gridsNeeded(databaseContext); + const auto l_gridsNeeded = operation->gridsNeeded( + databaseContext, considerKnownGridsAsAvailable); for (const auto &gridDesc : l_gridsNeeded) { res.insert(gridDesc); } @@ -11132,7 +11137,10 @@ struct FilterResults { bool gridsKnown = true; if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { hasGrids = true; if (gridAvailabilityUse == @@ -11254,6 +11262,7 @@ struct FilterResults { CoordinateOperationPtr lastOp; bool first = true; + const auto gridAvailabilityUse = context->getGridAvailabilityUse(); for (const auto &op : res) { const auto curAccuracy = getAccuracy(op); bool dummy = false; @@ -11266,7 +11275,10 @@ struct FilterResults { if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { curHasGrids = true; curSetOfGrids.insert(gridDesc.shortName); @@ -11571,6 +11583,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11590,9 +11603,16 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( tmpAuthFactory->createFromCoordinateReferenceSystemCodes( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, false, context.extent1, context.extent2); res.insert(res.end(), resTmp.begin(), resTmp.end()); @@ -11635,6 +11655,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( std::list> ids; buildCRSIds(targetCRS, context, ids); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &id : ids) { const auto &targetAuthName = id.first; const auto &targetCode = id.second; @@ -11648,9 +11669,15 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( auto res = tmpAuthFactory->createFromCoordinateReferenceSystemCodes( std::string(), std::string(), targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, true, context.extent1, context.extent2); if (!res.empty()) { @@ -11698,6 +11725,7 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11717,21 +11745,28 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( std::vector res; if (useCreateBetweenGeodeticCRSWithDatumBasedIntermediates) { - res = tmpAuthFactory - ->createBetweenGeodeticCRSWithDatumBasedIntermediates( - sourceCRS, srcAuthName, srcCode, targetCRS, - targetAuthName, targetCode, - context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext:: - GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, - context.context->getDiscardSuperseded(), - authFactory->getAuthority() != "any" && - authorities.size() > 1 - ? authorities - : std::vector(), - context.extent1, context.extent2); + res = + tmpAuthFactory + ->createBetweenGeodeticCRSWithDatumBasedIntermediates( + sourceCRS, srcAuthName, srcCode, targetCRS, + targetAuthName, targetCode, + context.context->getUsePROJAlternativeGridNames(), + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + context.context->getDiscardSuperseded(), + authFactory->getAuthority() != "any" && + authorities.size() > 1 + ? authorities + : std::vector(), + context.extent1, context.extent2); } else { io::AuthorityFactory::ObjectType intermediateObjectType = io::AuthorityFactory::ObjectType::CRS; @@ -11749,9 +11784,15 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( res = tmpAuthFactory->createFromCRSCodesWithIntermediates( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), context.context->getIntermediateCRS(), intermediateObjectType, @@ -14167,15 +14208,21 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( componentsSrc[1], targetCRS->promoteTo3D(std::string(), dbContext), context); bool foundRegisteredTransformWithAllGridsAvailable = false; + const auto gridAvailabilityUse = + context.context->getGridAvailabilityUse(); const bool ignoreMissingGrids = - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY; for (const auto &op : verticalTransforms) { if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -14204,7 +14251,11 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -15034,8 +15085,9 @@ CoordinateOperationNNPtr PROJBasedOperation::_shallowClone() const { // --------------------------------------------------------------------------- -std::set PROJBasedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set +PROJBasedOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set res; try { @@ -15048,7 +15100,8 @@ std::set PROJBasedOperation::gridsNeeded( desc.shortName = shortName; if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, desc.url, + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); diff --git a/src/iso19111/factory.cpp b/src/iso19111/factory.cpp index 57850303..dae8680c 100644 --- a/src/iso19111/factory.cpp +++ b/src/iso19111/factory.cpp @@ -1018,14 +1018,14 @@ bool DatabaseContext::lookForGridAlternative(const std::string &officialName, // --------------------------------------------------------------------------- -bool DatabaseContext::lookForGridInfo(const std::string &projFilename, - std::string &fullFilename, - std::string &packageName, - std::string &url, bool &directDownload, - bool &openLicense, - bool &gridAvailable) const { +bool DatabaseContext::lookForGridInfo( + const std::string &projFilename, bool considerKnownGridsAsAvailable, + std::string &fullFilename, std::string &packageName, std::string &url, + bool &directDownload, bool &openLicense, bool &gridAvailable) const { Private::GridInfoCache info; - if (d->getGridInfoFromCache(projFilename, info)) { + const std::string key(projFilename + + (considerKnownGridsAsAvailable ? "true" : "false")); + if (d->getGridInfoFromCache(key, info)) { fullFilename = info.fullFilename; packageName = info.packageName; url = info.url; @@ -1041,16 +1041,20 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = false; directDownload = false; - fullFilename.resize(2048); - if (d->pjCtxt() == nullptr) { - d->setPjCtxt(pj_get_default_ctx()); + if (considerKnownGridsAsAvailable) { + fullFilename = projFilename; + } else { + fullFilename.resize(2048); + if (d->pjCtxt() == nullptr) { + d->setPjCtxt(pj_get_default_ctx()); + } + int errno_before = proj_context_errno(d->pjCtxt()); + gridAvailable = + pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], + fullFilename.size() - 1) != 0; + proj_context_errno_set(d->pjCtxt(), errno_before); + fullFilename.resize(strlen(fullFilename.c_str())); } - int errno_before = proj_context_errno(d->pjCtxt()); - gridAvailable = - pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], - fullFilename.size() - 1) != 0; - proj_context_errno_set(d->pjCtxt(), errno_before); - fullFilename.resize(strlen(fullFilename.c_str())); auto res = d->run("SELECT " @@ -1074,6 +1078,10 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = (row[3].empty() ? row[4] : row[3]) == "1"; directDownload = (row[5].empty() ? row[6] : row[5]) == "1"; + if (considerKnownGridsAsAvailable && !packageName.empty()) { + gridAvailable = true; + } + info.fullFilename = fullFilename; info.packageName = packageName; info.url = url; @@ -1082,7 +1090,7 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, } info.gridAvailable = gridAvailable; info.found = ret; - d->cache(projFilename, info); + d->cache(key, info); return ret; } @@ -1264,8 +1272,8 @@ struct AuthorityFactory::Private { return AuthorityFactory::create(context_, auth_name); } - bool - rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op); + bool rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable); UnitOfMeasure createUnitOfMeasure(const std::string &auth_name, const std::string &code); @@ -1392,8 +1400,10 @@ util::PropertyMap AuthorityFactory::Private::createProperties( // --------------------------------------------------------------------------- bool AuthorityFactory::Private::rejectOpDueToMissingGrid( - const operation::CoordinateOperationNNPtr &op) { - for (const auto &gridDesc : op->gridsNeeded(context())) { + const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable) { + for (const auto &gridDesc : + op->gridsNeeded(context(), considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return true; } @@ -3381,7 +3391,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSCode, const std::string &targetCRSCode) const { return createFromCoordinateReferenceSystemCodes( d->authority(), sourceCRSCode, d->authority(), targetCRSCode, false, - false, false); + false, false, false); } // --------------------------------------------------------------------------- @@ -3410,6 +3420,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param tryReverseOrder whether to search in the reverse order too (and thus @@ -3430,8 +3442,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, bool tryReverseOrder, - bool reportOnlyIntersectingTransformations, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + bool tryReverseOrder, bool reportOnlyIntersectingTransformations, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -3442,6 +3454,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( cacheKey += targetCRSCode; cacheKey += (usePROJAlternativeGridNames ? '1' : '0'); cacheKey += (discardIfMissingGrid ? '1' : '0'); + cacheKey += (considerKnownGridsAsAvailable ? '1' : '0'); cacheKey += (discardSuperseded ? '1' : '0'); cacheKey += (tryReverseOrder ? '1' : '0'); cacheKey += (reportOnlyIntersectingTransformations ? '1' : '0'); @@ -3680,7 +3693,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( target_crs_code != targetCRSCode))) { op = op->inverse(); } - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -3745,6 +3759,8 @@ static bool useIrrelevantPivot(const operation::CoordinateOperationNNPtr &op, * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param intermediateCRSAuthCodes List of (auth_name, code) of CRS that can be @@ -3773,7 +3789,7 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector> &intermediateCRSAuthCodes, ObjectType allowedIntermediateObjectType, @@ -4221,7 +4237,8 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( std::vector list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -4239,7 +4256,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, const std::vector &allowedAuthorities, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + const std::vector &allowedAuthorities, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -4822,7 +4840,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( std::vector list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 6bdd5510..926505ba 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -203,6 +203,27 @@ static const char *get_path_from_win32_projlib(const char *name, std::string& ou /* pj_open_lib_internal() */ /************************************************************************/ +#ifdef WIN32 +static const char dir_chars[] = "/\\"; +static const char dirSeparator = ';'; +#else +static const char dir_chars[] = "/"; +static const char dirSeparator = ':'; +#endif + +static bool is_tilde_slash(const char* name) +{ + return *name == '~' && strchr(dir_chars,name[1]); +} + +static bool is_rel_or_absolute_filename(const char *name) +{ + return strchr(dir_chars,*name) + || (*name == '.' && strchr(dir_chars,name[1])) + || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) + || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])); +} + static void* pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, void* (*open_file)(projCtx, const char*, const char*), @@ -211,13 +232,6 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, std::string fname; const char *sysname = nullptr; void* fid = nullptr; -#ifdef WIN32 - static const char dir_chars[] = "/\\"; - const char dirSeparator = ';'; -#else - static const char dir_chars[] = "/"; - const char dirSeparator = ':'; -#endif if( ctx == nullptr ) { ctx = pj_get_default_ctx(); @@ -227,7 +241,7 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, out_full_filename[0] = '\0'; /* check if ~/name */ - if (*name == '~' && strchr(dir_chars,name[1]) ) + if (is_tilde_slash(name)) if ((sysname = getenv("HOME")) != nullptr) { fname = sysname; fname += DIR_CHAR; @@ -236,11 +250,8 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, } else return nullptr; - /* or fixed path: /name, ./name or ../name */ - else if (strchr(dir_chars,*name) - || (*name == '.' && strchr(dir_chars,name[1])) - || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) - || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) + /* or fixed path: /name, ./name or ../name or http[s]:// */ + else if (is_rel_or_absolute_filename(name) || starts_with(name, "http://") || starts_with(name, "https://")) sysname = name; @@ -344,11 +355,31 @@ static void* pj_open_file_with_manager(projCtx ctx, const char *name, std::unique_ptr NS_PROJ::FileManager::open_resource_file( projCtx ctx, const char *name) { - return std::unique_ptr( + auto file = std::unique_ptr( reinterpret_cast( pj_open_lib_internal(ctx, name, "rb", pj_open_file_with_manager, nullptr, 0))); + if( file == nullptr && + !is_tilde_slash(name) && + !is_rel_or_absolute_filename(name) && + !starts_with(name, "http://") && + !starts_with(name, "https://") && + pj_context_is_network_enabled(ctx) ) { + std::string remote_file("https://cdn.proj.org/"); + remote_file += name; + auto pos = remote_file.rfind('.'); + if( pos + 4 == remote_file.size() ) { + remote_file = remote_file.substr(0, pos) + ".tif"; + file = open(ctx, remote_file.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } + } + } + return file; } /************************************************************************/ diff --git a/src/proj.h b/src/proj.h index 4f7878a4..7210318c 100644 --- a/src/proj.h +++ b/src/proj.h @@ -697,6 +697,12 @@ typedef enum { /** Ignore grid availability at all. Results will be presented as if * all grids were available. */ PROJ_GRID_AVAILABILITY_IGNORED, + + /** Results will be presented as if grids known to PROJ (that is + * registered in the grid_alternatives table of its database) were + * available. Used typically when networking is enabled. + */ + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE, } PROJ_GRID_AVAILABILITY_USE; /** \brief PROJ string version. */ diff --git a/src/proj_internal.h b/src/proj_internal.h index d54d8fb9..983f1c07 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -697,6 +697,7 @@ struct projCtx_t { void* file_finder_user_data = nullptr; projNetworkCallbacksAndData networking{}; + bool defer_grid_opening = false; // set by pj_obj_create() int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() @@ -826,7 +827,8 @@ PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); -bool pj_context_is_network_enabled(PJ_CONTEXT* ctx); +// For use by projinfo +bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); /* classic public API */ #include "proj_api.h" diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index e9983df6..3b6e366f 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -17,6 +17,7 @@ struct hgridshiftData { double t_final = 0; double t_epoch = 0; ListOfHGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace @@ -25,6 +26,14 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -40,6 +49,14 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -114,9 +131,9 @@ PJ *TRANSFORMATION(hgridshift,0) { return destructor (P, PJD_ERR_NO_ARGS); } - /* TODO: Refactor into shared function that can be used */ - /* by both vgridshift and hgridshift */ - if (pj_param(P->ctx, P->params, "tt_final").i) { + /* TODO: Refactor into shared function that can be used */ + /* by both vgridshift and hgridshift */ + if (pj_param(P->ctx, P->params, "tt_final").i) { Q->t_final = pj_param (P->ctx, P->params, "dt_final").f; if (Q->t_final == 0) { /* a number wasn't passed to +t_final, let's see if it was "now" */ @@ -131,16 +148,21 @@ PJ *TRANSFORMATION(hgridshift,0) { } } - if (pj_param(P->ctx, P->params, "tt_epoch").i) + if (pj_param(P->ctx, P->params, "tt_epoch").i) Q->t_epoch = pj_param (P->ctx, P->params, "dt_epoch").f; - Q->grids = proj_hgrid_init(P, "grids"); - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "hgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; } + else { + Q->grids = proj_hgrid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "hgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + } return P; } diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index b964f45b..f35832e1 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -18,14 +18,51 @@ struct vgridshiftData { double t_epoch = 0; double forward_multiplier = 0; ListOfVGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace +static void deal_with_vertcon_gtx_hack(PJ *P) +{ + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; + // The .gtx VERTCON files stored millimetres, but the .tif files + // are in metres. + if( Q->forward_multiplier != 0.001 ) { + return; + } + const char* gridname = pj_param(P->ctx, P->params, "sgrids").s; + if( !gridname ) { + return; + } + if( strcmp(gridname, "vertconw.gtx") != 0 && + strcmp(gridname, "vertconc.gtx") != 0 && + strcmp(gridname, "vertcone.gtx") != 0 ) { + return; + } + if( Q->grids.empty() ) { + return; + } + const auto& grids = Q->grids[0]->grids(); + if( !grids.empty() && + grids[0]->name().find(".tif") != std::string::npos ) { + Q->forward_multiplier = 1.0; + } +} + static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -41,6 +78,15 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -132,13 +178,18 @@ PJ *TRANSFORMATION(vgridshift,0) { Q->forward_multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; } - /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ - Q->grids = proj_vgrid_init(P, "grids"); - - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "vgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ + Q->grids = proj_vgrid_init(P, "grids"); + + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "vgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } P->fwd4d = forward_4d; -- cgit v1.2.3 From 2093aca0720949303410280912b61efd791d2f01 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 20:13:03 +0100 Subject: Network: make CDN endpoint configurable either in proj.ini, PROJ_NETWORK_ENDPOINT or proj_context_set_url_endpoint() --- src/filemanager.cpp | 25 ++++++++++++ src/open_lib.cpp | 113 +++++++++++++++++++++++++++++++++++++++++++++++----- src/proj.h | 2 + src/proj_internal.h | 7 ++++ 4 files changed, 137 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index d9a02632..97d7369e 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -772,6 +772,9 @@ int proj_context_set_network_callbacks( /** Enable or disable network access. * +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK environment variable. +* * @param ctx PROJ context, or NULL * @param enable TRUE if network access is allowed. * @return TRUE if network access is possible. That is either libcurl is @@ -781,6 +784,8 @@ int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); ctx->networking.enabled_env_variable_checked = true; ctx->networking.enabled = enable != FALSE; #ifdef CURL_ENABLED @@ -793,6 +798,25 @@ int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { // --------------------------------------------------------------------------- +/** Define the URL endpoint to query for remote grids. +* +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK_ENDPOINT environment variable. +* +* @param ctx PROJ context, or NULL +* @param url Endpoint URL. Must NOT be NULL. +*/ +void proj_context_set_url_endpoint(PJ_CONTEXT *ctx, const char *url) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); + ctx->endpoint = url; +} + +// --------------------------------------------------------------------------- + //! @cond Doxygen_Suppress bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { @@ -808,6 +832,7 @@ bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { ci_equal(enabled, "YES") || ci_equal(enabled, "TRUE"); } + pj_load_ini(ctx); ctx->networking.enabled_env_variable_checked = true; return ctx->networking.enabled; } diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 926505ba..cde5be7b 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -366,16 +366,21 @@ std::unique_ptr NS_PROJ::FileManager::open_resource_file( !starts_with(name, "http://") && !starts_with(name, "https://") && pj_context_is_network_enabled(ctx) ) { - std::string remote_file("https://cdn.proj.org/"); - remote_file += name; - auto pos = remote_file.rfind('.'); - if( pos + 4 == remote_file.size() ) { - remote_file = remote_file.substr(0, pos) + ".tif"; - file = open(ctx, remote_file.c_str()); - if( file ) { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "Using %s", remote_file.c_str() ); - pj_ctx_set_errno( ctx, 0 ); + std::string remote_file(pj_context_get_url_endpoint(ctx)); + if( !remote_file.empty() ) { + if( remote_file.back() != '/' ) { + remote_file += '/'; + } + remote_file += name; + auto pos = remote_file.rfind('.'); + if( pos + 4 == remote_file.size() ) { + remote_file = remote_file.substr(0, pos) + ".tif"; + file = open(ctx, remote_file.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } } } } @@ -433,3 +438,91 @@ int pj_find_file(projCtx ctx, const char *short_filename, } return 0; } + +/************************************************************************/ +/* pj_context_get_url_endpoint() */ +/************************************************************************/ + +std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx) +{ + if( !ctx->endpoint.empty() ) { + return ctx->endpoint; + } + pj_load_ini(ctx); + return ctx->endpoint; +} + +/************************************************************************/ +/* trim() */ +/************************************************************************/ + +static std::string trim(const std::string& s) { + const auto first = s.find_first_not_of(' '); + const auto last = s.find_last_not_of(' '); + if( first == std::string::npos || last == std::string::npos ) { + return std::string(); + } + return s.substr(first, last - first + 1); +} + +/************************************************************************/ +/* pj_load_ini() */ +/************************************************************************/ + +void pj_load_ini(projCtx ctx) +{ + if( ctx->iniFileLoaded ) + return; + + const char* endpoint_from_env = getenv("PROJ_NETWORK_ENDPOINT"); + if( endpoint_from_env && endpoint_from_env[0] != '\0' ) { + ctx->endpoint = endpoint_from_env; + } + + ctx->iniFileLoaded = true; + auto file = std::unique_ptr( + reinterpret_cast( + pj_open_lib_internal(ctx, "proj.ini", "rb", + pj_open_file_with_manager, + nullptr, 0))); + if( !file ) + return; + file->seek(0, SEEK_END); + const auto filesize = file->tell(); + if( filesize == 0 || filesize > 100 * 1024U ) + return; + file->seek(0, SEEK_SET); + std::string content; + content.resize(static_cast(filesize)); + const auto nread = file->read(&content[0], content.size()); + if( nread != content.size() ) + return; + content += '\n'; + size_t pos = 0; + while( pos != std::string::npos ) { + const auto eol = content.find_first_of("\r\n", pos); + if( eol == std::string::npos ) { + break; + } + + const auto equal = content.find('=', pos); + if( equal < eol ) + { + const auto key = trim(content.substr(pos, equal-pos)); + const auto value = trim(content.substr(equal + 1, + eol - (equal+1))); + if( ctx->endpoint.empty() && key == "cdn_endpoint" ) { + ctx->endpoint = value; + } else if ( key == "network" ) { + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled == nullptr || enabled[0] == '\0') { + ctx->networking.enabled = ci_equal(value, "ON") || + ci_equal(value, "YES") || + ci_equal(value, "TRUE"); + } + } + } + + pos = content.find_first_not_of("\r\n", eol); + } +} diff --git a/src/proj.h b/src/proj.h index 7210318c..3f0879bb 100644 --- a/src/proj.h +++ b/src/proj.h @@ -424,6 +424,8 @@ int PROJ_DLL proj_context_set_network_callbacks( int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, int enabled); +void PROJ_DLL proj_context_set_url_endpoint(PJ_CONTEXT* ctx, const char* url); + /*! @cond Doxygen_Suppress */ /* Manage the transformation definition object PJ */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 983f1c07..669fd2b5 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -699,6 +699,9 @@ struct projCtx_t { projNetworkCallbacksAndData networking{}; bool defer_grid_opening = false; // set by pj_obj_create() + bool iniFileLoaded = false; + std::string endpoint{}; + int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() projCtx_t() = default; @@ -830,6 +833,10 @@ void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); // For use by projinfo bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); +std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx); + +void pj_load_ini(PJ_CONTEXT* ctx); + /* classic public API */ #include "proj_api.h" -- cgit v1.2.3 From aa8c7826cf17e650ee2c3a2281aca49db37c4e81 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 22:16:28 +0100 Subject: Network: rework error handling --- src/filemanager.cpp | 95 +++++++++++++++++++++++++++++++++++++++-------------- src/proj.h | 18 ++++++---- src/proj_internal.h | 1 - 3 files changed, 81 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 97d7369e..551301c6 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -323,11 +323,19 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; - auto handle = - ctx->networking.open(ctx, filename, 0, buffer.size(), &buffer[0], - &size_read, ctx->networking.user_data); + std::string errorBuffer; + errorBuffer.resize(1024); + + auto handle = ctx->networking.open( + ctx, filename, 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); buffer.resize(size_read); gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename, + errorBuffer.c_str()); + } unsigned long long filesize = 0; if (handle) { @@ -405,11 +413,14 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); size_t nRead = 0; + std::string errorBuffer; + errorBuffer.resize(1024); if (!m_handle) { m_handle = m_ctx->networking.open( m_ctx, m_url.c_str(), offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], - &nRead, m_ctx->networking.user_data); + &nRead, errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); if (!m_handle) { return 0; } @@ -417,9 +428,15 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { nRead = m_ctx->networking.read_range( m_ctx, m_handle, offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + errorBuffer.size(), &errorBuffer[0], m_ctx->networking.user_data); } if (nRead == 0) { + errorBuffer.resize(strlen(errorBuffer.data())); + if (!errorBuffer.empty()) { + pj_log(m_ctx, PJ_LOG_ERROR, "Cannot read in %s: %s", + m_url.c_str(), errorBuffer.c_str()); + } return 0; } region.resize(nRead); @@ -542,10 +559,10 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- -static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, - unsigned long long offset, - size_t size_to_read, void *buffer, - size_t *out_size_read, void *) { +static PROJ_NETWORK_HANDLE * +pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *) { CURL *hCurlHandle = curl_easy_init(); if (!hCurlHandle) return nullptr; @@ -583,6 +600,10 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -594,10 +615,24 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } curl_easy_cleanup(hCurlHandle); return nullptr; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -619,7 +654,8 @@ static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, unsigned long long offset, size_t size_to_read, - void *buffer, void *) { + void *buffer, size_t error_string_max_size, + char *out_error_string, void *) { auto handle = reinterpret_cast(raw_handle); auto hCurlHandle = handle->m_handle; @@ -633,6 +669,10 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -641,9 +681,23 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } return 0; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -682,8 +736,12 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, size_t, /* size to read */ void *, /* buffer to update with bytes read*/ size_t *, /* output: size actually read */ + size_t error_string_max_size, char *out_error_string, void * /*user_data*/) { - pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); + if (out_error_string) { + snprintf(out_error_string, error_string_max_size, "%s", + "Network functionality not available"); + } return nullptr; } @@ -692,14 +750,6 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void * /*user_data*/) {} -// --------------------------------------------------------------------------- - -static const char *no_op_network_get_last_error(PJ_CONTEXT *, - PROJ_NETWORK_HANDLE *, - void * /*user_data*/) { - return "Network functionality not available"; -} - #endif // --------------------------------------------------------------------------- @@ -713,7 +763,6 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { #else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; - ctx->networking.get_last_error = no_op_network_get_last_error; #endif } @@ -741,7 +790,6 @@ NS_PROJ_END * @param close_cbk Callback to close a remote file. * @param get_header_value_cbk Callback to get HTTP headers * @param read_range_cbk Callback to read a range of bytes inside a remote file. - * @param get_last_error_cbk Callback to get last error message. * @param user_data Arbitrary pointer provided by the user, and passed to the * above callbacks. May be NULL. * @return TRUE in case of success. @@ -750,20 +798,17 @@ int proj_context_set_network_callbacks( PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_read_range_type read_range_cbk, - proj_network_get_last_error_type get_last_error_cbk, void *user_data) { + proj_network_read_range_type read_range_cbk, void *user_data) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } - if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk || - !get_last_error_cbk) { + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk) { return false; } ctx->networking.open = open_cbk; ctx->networking.close = close_cbk; ctx->networking.get_header_value = get_header_value_cbk; ctx->networking.read_range = read_range_cbk; - ctx->networking.get_last_error = get_last_error_cbk; ctx->networking.user_data = user_data; return true; } diff --git a/src/proj.h b/src/proj.h index 3f0879bb..96b9c3f8 100644 --- a/src/proj.h +++ b/src/proj.h @@ -369,6 +369,9 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; * headers from the server response to be able to respond to * proj_network_get_header_value_cbk_type callback. * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * * @return a non-NULL opaque handle in case of success. */ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( @@ -378,6 +381,8 @@ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( size_t size_to_read, void* buffer, size_t* out_size_read, + size_t error_string_max_size, + char* out_error_string, void* user_data); /** Network access: close callback */ @@ -396,6 +401,10 @@ typedef const char* (*proj_network_get_header_value_cbk_type)( * * Read size_to_read bytes from handle, starting at offset, into * buffer. + * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * * @return the number of bytes actually read (0 in case of error) */ typedef size_t (*proj_network_read_range_type)( @@ -404,12 +413,8 @@ typedef size_t (*proj_network_read_range_type)( unsigned long long offset, size_t size_to_read, void* buffer, - void* user_data); - -/** Network access: get last error message */ -typedef const char* (*proj_network_get_last_error_type)( - PJ_CONTEXT* ctx, - PROJ_NETWORK_HANDLE*, + size_t error_string_max_size, + char* out_error_string, void* user_data); int PROJ_DLL proj_context_set_network_callbacks( @@ -418,7 +423,6 @@ int PROJ_DLL proj_context_set_network_callbacks( proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, proj_network_read_range_type read_range_cbk, - proj_network_get_last_error_type get_last_error_cbk, void* user_data); int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, diff --git a/src/proj_internal.h b/src/proj_internal.h index 669fd2b5..27b30954 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -674,7 +674,6 @@ struct projNetworkCallbacksAndData proj_network_close_cbk_type close = nullptr; proj_network_get_header_value_cbk_type get_header_value = nullptr; proj_network_read_range_type read_range = nullptr; - proj_network_get_last_error_type get_last_error = nullptr; void* user_data = nullptr; }; -- cgit v1.2.3 From f085b39752d00a296c288be42dfc69b39b73823f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 12:25:03 +0100 Subject: Handle context reassignment for Grid/GridSet/File objects --- src/ctx.cpp | 4 +- src/filemanager.cpp | 18 +++++++- src/filemanager.hpp | 1 + src/grids.cpp | 88 ++++++++++++++++++++++++++++++++++++ src/grids.hpp | 12 +++++ src/pipeline.cpp | 4 +- src/proj_internal.h | 4 +- src/transformations/hgridshift.cpp | 9 ++++ src/transformations/vgridshift.cpp | 10 ++++ src/transformations/xyzgridshift.cpp | 10 ++++ 10 files changed, 152 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/ctx.cpp b/src/ctx.cpp index a7cf8cb3..6172b3c8 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -61,9 +61,9 @@ void pj_set_ctx( projPJ pj, projCtx ctx ) if (pj==nullptr) return; pj->ctx = ctx; - if( pj->is_pipeline ) + if( pj->reassign_context ) { - pj_pipeline_assign_context_to_steps(pj, ctx); + pj->reassign_context(pj, ctx); } for( const auto &alt: pj->alternativeCoordinateOperations ) { diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 551301c6..dabb46e0 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -94,6 +94,7 @@ class FileStdio : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence = SEEK_SET) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -156,6 +157,7 @@ class FileLegacyAdapter : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence = SEEK_SET) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -286,6 +288,7 @@ class NetworkFile : public File { size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; unsigned long long m_filesize; + proj_network_close_cbk_type m_closeCbk; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; @@ -296,7 +299,8 @@ class NetworkFile : public File { unsigned long long lastDownloadOffset, unsigned long long filesize) : File(url), m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} + m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize), + m_closeCbk(ctx->networking.close) {} public: ~NetworkFile() override; @@ -304,6 +308,7 @@ class NetworkFile : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override; static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -500,6 +505,17 @@ NetworkFile::~NetworkFile() { // --------------------------------------------------------------------------- +void NetworkFile::reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + if (m_closeCbk != m_ctx->networking.close) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Networking close callback has changed following context " + "reassignment ! This is highly suspicious"); + } +} + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifndef REMOVE_LEGACY_SUPPORT // If the user has specified a legacy fileapi, use it diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 46391597..972634c2 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -68,6 +68,7 @@ class File { virtual size_t read(void *buffer, size_t sizeBytes) = 0; virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; virtual unsigned long long tell() = 0; + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; const std::string &name() const { return name_; } }; diff --git a/src/grids.cpp b/src/grids.cpp index 45e7e8b0..a3d984de 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -139,6 +139,7 @@ class NullVerticalShiftGrid : public VerticalShiftGrid { bool isNullGrid() const override { return true; } bool valueAt(int, int, float &out) const override; bool isNodata(float, double) const override { return false; } + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- @@ -171,6 +172,11 @@ class GTXVerticalShiftGrid : public VerticalShiftGrid { static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &name); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -412,6 +418,8 @@ class GTiffGrid : public Grid { std::string metadataItem(const std::string &key, int sample = -1) const; uint32 subfileType() const { return m_subfileType; } + + void reassign_context(PJ_CONTEXT *ctx) { m_ctx = ctx; } }; // --------------------------------------------------------------------------- @@ -738,6 +746,11 @@ class GTiffDataset { bool openTIFF(const std::string &filename); std::unique_ptr nextGrid(); + + void reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1056,6 +1069,11 @@ class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + VerticalShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } }; #endif // TIFF_ENABLED @@ -1150,6 +1168,10 @@ class GTiffVGrid : public VerticalShiftGrid { } void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1374,6 +1396,14 @@ const VerticalShiftGrid *VerticalShiftGridSet::gridAt(double lon, // --------------------------------------------------------------------------- +void VerticalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + HorizontalShiftGrid::HorizontalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) @@ -1402,6 +1432,8 @@ class NullHorizontalShiftGrid : public HorizontalShiftGrid { bool isNullGrid() const override { return true; } bool valueAt(int, int, float &lonShift, float &latShift) const override; + + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- @@ -1443,6 +1475,11 @@ class NTv1Grid : public HorizontalShiftGrid { static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1556,6 +1593,11 @@ class CTable2Grid : public HorizontalShiftGrid { static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1654,6 +1696,11 @@ class NTv2GridSet : public HorizontalShiftGridSet { static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1679,6 +1726,11 @@ class NTv2Grid : public HorizontalShiftGrid { m_mustSwap(mustSwapIn) {} bool valueAt(int, int, float &lonShift, float &latShift) const override; + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1857,6 +1909,11 @@ class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1884,6 +1941,10 @@ class GTiffHGrid : public HorizontalShiftGrid { bool valueAt(int x, int y, float &lonShift, float &latShift) const override; void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -2235,6 +2296,14 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, return nullptr; } +// --------------------------------------------------------------------------- + +void HorizontalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + #ifdef TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2250,6 +2319,11 @@ class GTiffGenericGridShiftSet : public GenericShiftGridSet, static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + GenericShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -2287,6 +2361,10 @@ class GTiffGenericGrid : public GenericShiftGrid { void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -2356,6 +2434,8 @@ class NullGenericShiftGrid : public GenericShiftGrid { std::string metadataItem(const std::string &, int) const override { return std::string(); } + + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- @@ -2516,6 +2596,14 @@ const GenericShiftGrid *GenericShiftGridSet::gridAt(double lon, // --------------------------------------------------------------------------- +void GenericShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { std::string key("s"); key += gridkey; diff --git a/src/grids.hpp b/src/grids.hpp index 0f595754..fde3eb3e 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -88,6 +88,8 @@ class VerticalShiftGrid : public Grid { // x = 0 is western-most column, y = 0 is southern-most line virtual bool valueAt(int x, int y, float &out) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- @@ -112,6 +114,8 @@ class VerticalShiftGridSet { return m_grids; } const VerticalShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -130,6 +134,8 @@ class HorizontalShiftGrid : public Grid { // x = 0 is western-most column, y = 0 is southern-most line virtual bool valueAt(int x, int y, float &lonShift, float &latShift) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- @@ -154,6 +160,8 @@ class HorizontalShiftGridSet { return m_grids; } const HorizontalShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -181,6 +189,8 @@ class GenericShiftGrid : public Grid { // x = 0 is western-most column, y = 0 is southern-most line virtual bool valueAt(int x, int y, int sample, float &out) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- @@ -205,6 +215,8 @@ class GenericShiftGridSet { return m_grids; } const GenericShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- diff --git a/src/pipeline.cpp b/src/pipeline.cpp index 96767143..f65dbfa0 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -155,7 +155,7 @@ static PJ_LPZ pipeline_reverse_3d (PJ_XYZ xyz, PJ *P); static PJ_XY pipeline_forward (PJ_LP lp, PJ *P); static PJ_LP pipeline_reverse (PJ_XY xy, PJ *P); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ) +static void pipeline_reassign_context( PJ* P, PJ_CONTEXT* ctx ) { auto pipeline = static_cast(P->opaque); for( auto& step: pipeline->steps ) @@ -413,7 +413,7 @@ PJ *OPERATION(pipeline,0) { P->fwd = pipeline_forward; P->inv = pipeline_reverse; P->destructor = destructor; - P->is_pipeline = 1; + P->reassign_context = pipeline_reassign_context; /* Currently, the pipeline driver is a raw bit mover, enabling other operations */ /* to collaborate efficiently. All prep/fin stuff is done at the step levels. */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 27b30954..12ada034 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -343,6 +343,7 @@ struct PJconsts { PJ_OPERATOR inv4d = nullptr; PJ_DESTRUCTOR destructor = nullptr; + void (*reassign_context)(PJ*, projCtx_t *) = nullptr; /************************************************************************************* @@ -410,7 +411,6 @@ struct PJconsts { int geoc = 0; /* Geocentric latitude flag */ int is_latlong = 0; /* proj=latlong ... not really a projection at all */ int is_geocent = 0; /* proj=geocent ... not really a projection at all */ - int is_pipeline = 0; /* 1 if PJ represents a pipeline */ int need_ellps = 0; /* 0 for operations that are purely cartesian */ int skip_fwd_prepare = 0; int skip_fwd_finalize = 0; @@ -827,8 +827,6 @@ std::string pj_double_quote_string_param_if_needed(const std::string& str); PJ *pj_create_internal (PJ_CONTEXT *ctx, const char *definition); PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); - // For use by projinfo bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index 3b6e366f..24da4dde 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -111,10 +111,19 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct hgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + PJ *TRANSFORMATION(hgridshift,0) { auto Q = new hgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; P->fwd4d = forward_4d; P->inv4d = reverse_4d; diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index f35832e1..3e7a015e 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -142,10 +142,20 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct vgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + + PJ *TRANSFORMATION(vgridshift,0) { auto Q = new vgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; if (!pj_param(P->ctx, P->params, "tgrids").i) { proj_log_error(P, "vgridshift: +grids parameter missing."); diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index f23c2588..734ffc5d 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -270,10 +270,20 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct xyzgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + + PJ *TRANSFORMATION(xyzgridshift,0) { auto Q = new xyzgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; P->fwd4d = nullptr; P->inv4d = nullptr; -- cgit v1.2.3 From be596500398778c6d0fc31c0c15aa8b9df7aa3db Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 12:34:24 +0100 Subject: xyzgridshift: implement defered grid loading in network context --- src/transformations/xyzgridshift.cpp | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index 734ffc5d..a76f3255 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -46,6 +46,7 @@ struct xyzgridshiftData { PJ *cart = nullptr; bool grid_ref_is_input = true; ListOfGenericGrids grids{}; + bool defer_grid_opening = false; double multiplier = 1.0; }; } // anonymous namespace @@ -68,12 +69,20 @@ static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, // --------------------------------------------------------------------------- static bool get_grid_values(PJ* P, - const xyzgridshiftData* Q, + xyzgridshiftData* Q, const PJ_LP& lp, double& dx, double& dy, double& dz) { + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_generic_grid_init(P, "grids"); + if ( proj_errno(P) ) { + return false; + } + } + auto grid = findGrid(Q->grids, lp); if( !grid ) { return false; @@ -169,7 +178,7 @@ static bool get_grid_values(PJ* P, // --------------------------------------------------------------------------- static PJ_COORD iterative_adjustment(PJ* P, - const xyzgridshiftData* Q, + xyzgridshiftData* Q, const PJ_COORD& pointInit, double factor) { @@ -204,7 +213,7 @@ static PJ_COORD iterative_adjustment(PJ* P, // --------------------------------------------------------------------------- static PJ_COORD direct_adjustment(PJ* P, - const xyzgridshiftData* Q, + xyzgridshiftData* Q, PJ_COORD point, double factor) { @@ -328,11 +337,16 @@ PJ *TRANSFORMATION(xyzgridshift,0) { Q->multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; } - Q->grids = proj_generic_grid_init(P, "grids"); - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "xyzgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + Q->grids = proj_generic_grid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "xyzgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } return P; -- cgit v1.2.3 From f934c002c0742dc4bb6fcda1ff44e4035f472ce8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 17:41:33 +0100 Subject: CurlFileHandle: code improvement. No functional change --- src/filemanager.cpp | 96 +++++++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 43 deletions(-) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index dabb46e0..a279a6cf 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -542,20 +542,52 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifdef CURL_ENABLED struct CurlFileHandle { - CURL *m_handle = nullptr; - std::string m_headers; + std::string m_url; + CURL *m_handle; + std::string m_headers{}; std::string m_lastval{}; + char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; CurlFileHandle(const CurlFileHandle &) = delete; CurlFileHandle &operator=(const CurlFileHandle &) = delete; - explicit CurlFileHandle(CURL *handle, std::string &&headers) - : m_handle(handle), m_headers(std::move(headers)) {} + explicit CurlFileHandle(const char *url, CURL *handle); ~CurlFileHandle(); + + static PROJ_NETWORK_HANDLE * + open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *); }; // --------------------------------------------------------------------------- +CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) + : m_url(url), m_handle(handle) { + curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(handle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); +} + +// --------------------------------------------------------------------------- + CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } // --------------------------------------------------------------------------- @@ -575,31 +607,18 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- -static PROJ_NETWORK_HANDLE * -pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, - size_t size_to_read, void *buffer, size_t *out_size_read, - size_t error_string_max_size, char *out_error_string, void *) { +PROJ_NETWORK_HANDLE *CurlFileHandle::open(PJ_CONTEXT *, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *) { CURL *hCurlHandle = curl_easy_init(); if (!hCurlHandle) return nullptr; - curl_easy_setopt(hCurlHandle, CURLOPT_URL, url); - - if (getenv("PROJ_CURL_VERBOSE")) - curl_easy_setopt(hCurlHandle, CURLOPT_VERBOSE, 1); -// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. -#if LIBCURL_VERSION_NUM >= 0x073600 - curl_easy_setopt(hCurlHandle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); -#endif - - // Enable following redirections. Requires libcurl 7.10.1 at least. - curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(hCurlHandle, CURLOPT_MAXREDIRS, 10); - - if (getenv("PROJ_UNSAFE_SSL")) { - curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L); - } + auto file = + std::unique_ptr(new CurlFileHandle(url, hCurlHandle)); char szBuffer[128]; sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, @@ -616,9 +635,7 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); - char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; - szCurlErrBuf[0] = '\0'; - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + file->m_szCurlErrBuf[0] = '\0'; curl_easy_perform(hCurlHandle); @@ -631,19 +648,16 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); - if (response_code == 0 || response_code >= 300) { if (out_error_string) { - if (szCurlErrBuf[0]) { + if (file->m_szCurlErrBuf[0]) { snprintf(out_error_string, error_string_max_size, "%s", - szCurlErrBuf); + file->m_szCurlErrBuf); } else { snprintf(out_error_string, error_string_max_size, "HTTP error %ld: %s", response_code, body.c_str()); } } - curl_easy_cleanup(hCurlHandle); return nullptr; } if (out_error_string && error_string_max_size) { @@ -655,8 +669,8 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, } *out_size_read = std::min(size_to_read, body.size()); - return reinterpret_cast( - new CurlFileHandle(hCurlHandle, std::move(headers))); + file->m_headers = std::move(headers); + return reinterpret_cast(file.release()); } // --------------------------------------------------------------------------- @@ -685,9 +699,7 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); - char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; - szCurlErrBuf[0] = '\0'; - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + handle->m_szCurlErrBuf[0] = '\0'; curl_easy_perform(hCurlHandle); @@ -697,13 +709,11 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); - if (response_code == 0 || response_code >= 300) { if (out_error_string) { - if (szCurlErrBuf[0]) { + if (handle->m_szCurlErrBuf[0]) { snprintf(out_error_string, error_string_max_size, "%s", - szCurlErrBuf); + handle->m_szCurlErrBuf); } else { snprintf(out_error_string, error_string_max_size, "HTTP error %ld: %s", response_code, body.c_str()); @@ -772,7 +782,7 @@ static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { #ifdef CURL_ENABLED - ctx->networking.open = pj_curl_open; + ctx->networking.open = CurlFileHandle::open; ctx->networking.close = pj_curl_close; ctx->networking.read_range = pj_curl_read_range; ctx->networking.get_header_value = pj_curl_get_header_value; -- cgit v1.2.3 From 28e1770f27bb335d29bfa44a5c963904007b5e73 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 17:42:04 +0100 Subject: CurlFileHandle: set UserAgent --- src/filemanager.cpp | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) (limited to 'src') diff --git a/src/filemanager.cpp b/src/filemanager.cpp index a279a6cf..625dca03 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -61,8 +61,22 @@ class MyMutex { #include // for sqlite3_snprintf #endif +#if defined(__linux) +#include +#elif defined(_WIN32) +#include +#elif defined(__MACH__) && defined(__APPLE__) +#include +#elif defined(__FreeBSD__) +#include +#include +#endif + //! @cond Doxygen_Suppress +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + using namespace NS_PROJ::internal; NS_PROJ_START @@ -546,6 +560,7 @@ struct CurlFileHandle { CURL *m_handle; std::string m_headers{}; std::string m_lastval{}; + std::string m_useragent{}; char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; CurlFileHandle(const CurlFileHandle &) = delete; @@ -562,6 +577,67 @@ struct CurlFileHandle { // --------------------------------------------------------------------------- +static std::string GetExecutableName() { +#if defined(__linux) + std::string path; + path.resize(1024); + const auto ret = readlink("/proc/self/exe", &path[0], path.size()); + if (ret > 0) { + path.resize(ret); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(_WIN32) + std::string path; + path.resize(1024); + if (GetModuleFileNameA(nullptr, &path[0], + static_cast(path.size()))) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('\\'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__MACH__) && defined(__APPLE__) + std::string path; + path.resize(1024); + uint32_t size = static_cast(path.size()); + if (_NSGetExecutablePath(&path[0], &size) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__FreeBSD__) + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PATHNAME; + mib[3] = -1; + std::string path; + path.resize(1024); + size_t size = path.size(); + if (sysctl(mib, 4, &path[0], &size, nullptr, 0) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#endif + + return std::string(); +} + +// --------------------------------------------------------------------------- + CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) : m_url(url), m_handle(handle) { curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); @@ -584,6 +660,16 @@ CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) } curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); + + if (getenv("PROJ_NO_USERAGENT") == nullptr) { + m_useragent = "PROJ " STR(PROJ_VERSION_MAJOR) "." STR( + PROJ_VERSION_MINOR) "." STR(PROJ_VERSION_PATCH); + const auto exeName = GetExecutableName(); + if (!exeName.empty()) { + m_useragent = exeName + " using " + m_useragent; + } + curl_easy_setopt(handle, CURLOPT_USERAGENT, m_useragent.data()); + } } // --------------------------------------------------------------------------- -- cgit v1.2.3 From 830f94a8117eff270acd2ef928850a9de5e164a9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 29 Dec 2019 23:57:23 +0100 Subject: proj_hgrid_value(): do not apply compensation for west-oriented longitude_offset. Fixes regression due to grid refactoring work on proj=deformation use case --- src/apply_gridshift.cpp | 16 ++++++++-------- src/grids.cpp | 38 ++++++++++++++++++++++++-------------- src/grids.hpp | 4 ++-- 3 files changed, 34 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp index b994474b..4ef86fc0 100644 --- a/src/apply_gridshift.cpp +++ b/src/apply_gridshift.cpp @@ -119,7 +119,7 @@ ListOfHGrids proj_hgrid_init(PJ* P, const char *gridkey) { typedef struct { pj_int32 lam, phi; } ILP; -static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { +static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid, bool compensateNTConvention) { PJ_LP val, frct; ILP indx; int in; @@ -164,10 +164,10 @@ static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { float f10Lon = 0, f10Lat = 0; float f01Lon = 0, f01Lat = 0; float f11Lon = 0, f11Lat = 0; - if( !grid->valueAt(indx.lam, indx.phi, f00Lon, f00Lat) || - !grid->valueAt(indx.lam + 1, indx.phi, f10Lon, f10Lat) || - !grid->valueAt(indx.lam, indx.phi + 1, f01Lon, f01Lat) || - !grid->valueAt(indx.lam + 1, indx.phi + 1, f11Lon, f11Lat) ) + if( !grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, f00Lat) || + !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, f10Lat) || + !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, f01Lat) || + !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, f11Lon, f11Lat) ) { return val; } @@ -210,7 +210,7 @@ PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* gri tb.lam = adjlon (tb.lam - M_PI) + M_PI; - t = nad_intr (tb, grid); + t = nad_intr (tb, grid, true); if (t.lam == HUGE_VAL) return t; @@ -224,7 +224,7 @@ PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* gri t.phi = tb.phi - t.phi; do { - del = nad_intr(t, grid); + del = nad_intr(t, grid, true); /* We can possibly go outside of the initial guessed grid, so try */ /* to fetch a new grid into which iterate... */ @@ -297,7 +297,7 @@ PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids& grids, PJ_LP lp) { lp.lam = adjlon(lp.lam - M_PI) + M_PI; - out = nad_intr(lp, grid); + out = nad_intr(lp, grid, false); if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); diff --git a/src/grids.cpp b/src/grids.cpp index a3d984de..5a99106b 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -1431,14 +1431,15 @@ class NullHorizontalShiftGrid : public HorizontalShiftGrid { bool isNullGrid() const override { return true; } - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- -bool NullHorizontalShiftGrid::valueAt(int, int, float &lonShift, +bool NullHorizontalShiftGrid::valueAt(int, int, bool, float &lonShift, float &latShift) const { lonShift = 0.0f; latShift = 0.0f; @@ -1471,7 +1472,8 @@ class NTv1Grid : public HorizontalShiftGrid { ~NTv1Grid() override; - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); @@ -1549,7 +1551,8 @@ NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, // --------------------------------------------------------------------------- -bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool NTv1Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); double two_doubles[2]; @@ -1566,7 +1569,8 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { /* convert seconds to radians */ latShift = static_cast(two_doubles[0] * ((M_PI / 180.0) / 3600.0)); // west longitude positive convention ! - lonShift = -static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); return true; } @@ -1589,7 +1593,8 @@ class CTable2Grid : public HorizontalShiftGrid { ~CTable2Grid() override; - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); @@ -1659,8 +1664,8 @@ CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, // --------------------------------------------------------------------------- -bool CTable2Grid::valueAt(int x, int y, float &lonShift, - float &latShift) const { +bool CTable2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_floats[2]; @@ -1675,7 +1680,7 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, latShift = two_floats[1]; // west longitude positive convention ! - lonShift = -two_floats[0]; + lonShift = (compensateNTConvention ? -1 : 1) * two_floats[0]; return true; } @@ -1725,7 +1730,8 @@ class NTv2Grid : public HorizontalShiftGrid { m_name(nameIn), m_ctx(ctx), m_fp(fp), m_offset(offsetIn), m_mustSwap(mustSwapIn) {} - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; @@ -1735,7 +1741,8 @@ class NTv2Grid : public HorizontalShiftGrid { // --------------------------------------------------------------------------- -bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool NTv2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_float[2]; @@ -1755,7 +1762,8 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { /* convert seconds to radians */ latShift = static_cast(two_float[0] * ((M_PI / 180.0) / 3600.0)); // west longitude positive convention ! - lonShift = -static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); return true; } @@ -1938,7 +1946,8 @@ class GTiffHGrid : public HorizontalShiftGrid { ~GTiffHGrid() override; - bool valueAt(int x, int y, float &lonShift, float &latShift) const override; + bool valueAt(int x, int y, bool, float &lonShift, + float &latShift) const override; void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); @@ -1968,7 +1977,8 @@ GTiffHGrid::~GTiffHGrid() = default; // --------------------------------------------------------------------------- -bool GTiffHGrid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool GTiffHGrid::valueAt(int x, int y, bool, float &lonShift, + float &latShift) const { if (!m_grid->valueAt(m_idxLatShift, x, y, latShift) || !m_grid->valueAt(m_idxLonShift, x, y, lonShift)) { return false; diff --git a/src/grids.hpp b/src/grids.hpp index fde3eb3e..aa852ef6 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -132,8 +132,8 @@ class HorizontalShiftGrid : public Grid { const HorizontalShiftGrid *gridAt(double lon, double lat) const; // x = 0 is western-most column, y = 0 is southern-most line - virtual bool valueAt(int x, int y, float &lonShift, - float &latShift) const = 0; + virtual bool valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const = 0; virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; -- cgit v1.2.3