From 1b702a82f5e009745f5ca8bbed92579e33f598d4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 6 Dec 2019 20:23:54 +0100 Subject: Build and CI: add libtiff dependency --- CMakeLists.txt | 9 +++++++++ appveyor.yml | 4 +++- configure.ac | 16 ++++++++++++++++ src/Makefile.am | 4 ++-- src/lib_proj.cmake | 4 ++-- travis/csa/before_install.sh | 2 +- travis/linux_clang/before_install.sh | 2 +- travis/linux_gcc/before_install.sh | 1 + travis/linux_gcc7/before_install.sh | 3 ++- travis/mingw32/install.sh | 19 +++++++++++++++++++ travis/osx/before_install.sh | 3 ++- 11 files changed, 58 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5715389..bb8ae88f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,6 +133,15 @@ IF("${SQLITE3_VERSION}" VERSION_LESS "3.11") message(SEND_ERROR "sqlite3 >= 3.11 required!") ENDIF() +################################################################################ +# Check for libtiff +################################################################################ + +find_package(TIFF REQUIRED) +if(NOT TIFF_FOUND) + message(SEND_ERROR "libtiff dependency not found!") +endif() + ################################################################################ # threading configuration ################################################################################ diff --git a/appveyor.yml b/appveyor.yml index a631c3e8..fb934f3a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,9 +27,9 @@ build_script: - set PATH=%CD%;%PATH% - cd .. - vcpkg install sqlite3:"%platform%"-windows + - vcpkg install tiff:"%platform%"-windows - set SQLITE3_BIN=%APPVEYOR_BUILD_FOLDER%\sqlite3\bin - mkdir %SQLITE3_BIN% - - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\sqlite3.dll %SQLITE3_BIN% - ps: | appveyor DownloadFile https://sqlite.org/2018/sqlite-tools-win32-x86-3250100.zip 7z x sqlite-tools-win32-x86-3250100.zip @@ -47,6 +47,8 @@ build_script: - set PROJ_DIR=%APPVEYOR_BUILD_FOLDER%\proj_dir - cmake -G "%VS_FULL%" .. -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBPROJ_SHARED="%BUILD_LIBPROJ_SHARED%" -DCMAKE_C_FLAGS="/WX" -DCMAKE_CXX_FLAGS="/WX" -DCMAKE_TOOLCHAIN_FILE=C:/projects/proj/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX="%PROJ_DIR%" - cmake --build . --config Release --target install + - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\*.dll %PROJ_DIR%\bin + - dir %PROJ_DIR%\bin test_script: - echo test_script diff --git a/configure.ac b/configure.ac index 0ea8e04a..7c04f60b 100644 --- a/configure.ac +++ b/configure.ac @@ -20,6 +20,8 @@ AC_PROG_LN_S AC_PROG_MAKE_SET AM_PROG_LIBTOOL +PKG_PROG_PKG_CONFIG + dnl Enable as much warnings as possible AX_CFLAGS_WARN_ALL(C_WFLAGS) AX_CXXFLAGS_WARN_ALL(CXX_WFLAGS) @@ -294,6 +296,20 @@ if test x"$SQLITE3_CHECK" != x"yes" ; then AC_MSG_ERROR([Please install sqlite3 binary.]) fi +dnl --------------------------------------------------------------------------- +dnl Check for libtiff +dnl --------------------------------------------------------------------------- + +if test "x$TIFF_CFLAGS$TIFF_LIBS" = "x" ; then + if $PKG_CONFIG libtiff; then + PKG_CHECK_MODULES([TIFF], [libtiff]) + else + PKG_CHECK_MODULES([TIFF], [libtiff-4]) + fi +fi +AC_SUBST(TIFF_CFLAGS,$TIFF_CFLAGS) +AC_SUBST(TIFF_LIBS,$TIFF_LIBS) + dnl --------------------------------------------------------------------------- dnl Check for external Google Test dnl --------------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index a12de4e1..f58e57fd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ + -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ @@ -43,7 +43,7 @@ geodtest_LDADD = libproj.la lib_LTLIBRARIES = libproj.la libproj_la_LDFLAGS = -no-undefined -version-info 17:0:2 -libproj_la_LIBADD = @SQLITE3_LIBS@ +libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ libproj_la_SOURCES = \ pj_list.h proj_internal.h \ diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index f6112aef..f1547afe 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -435,8 +435,8 @@ if(USE_THREAD AND Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) target_link_libraries(${PROJ_CORE_TARGET} ${CMAKE_THREAD_LIBS_INIT}) endif() -include_directories(${SQLITE3_INCLUDE_DIR}) -target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY}) +include_directories(${SQLITE3_INCLUDE_DIR} ${TIFF_INCLUDE_DIR}) +target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY} ${TIFF_LIBRARY}) if(MSVC) target_compile_definitions(${PROJ_CORE_TARGET} diff --git a/travis/csa/before_install.sh b/travis/csa/before_install.sh index c2e29064..8d0f3fd2 100755 --- a/travis/csa/before_install.sh +++ b/travis/csa/before_install.sh @@ -5,7 +5,7 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev CLANG_LLVM=clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-16.04 wget http://releases.llvm.org/6.0.0/$CLANG_LLVM.tar.xz diff --git a/travis/linux_clang/before_install.sh b/travis/linux_clang/before_install.sh index 8ce465a2..cfe7ba67 100755 --- a/travis/linux_clang/before_install.sh +++ b/travis/linux_clang/before_install.sh @@ -5,4 +5,4 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev diff --git a/travis/linux_gcc/before_install.sh b/travis/linux_gcc/before_install.sh index 7725b3ef..d4deb85e 100755 --- a/travis/linux_gcc/before_install.sh +++ b/travis/linux_gcc/before_install.sh @@ -9,6 +9,7 @@ sudo apt-get install -qq \ lcov \ doxygen graphviz \ sqlite3 libsqlite3-dev \ + libtiff-dev \ cppcheck scripts/cppcheck.sh diff --git a/travis/linux_gcc7/before_install.sh b/travis/linux_gcc7/before_install.sh index a3f6c8c0..b3e053c0 100755 --- a/travis/linux_gcc7/before_install.sh +++ b/travis/linux_gcc7/before_install.sh @@ -8,7 +8,8 @@ set -e sudo apt-get install -qq \ lcov \ doxygen graphviz \ - sqlite3 libsqlite3-dev + sqlite3 libsqlite3-dev \ + libtiff-dev #scripts/cppcheck.sh #scripts/doxygen.sh diff --git a/travis/mingw32/install.sh b/travis/mingw32/install.sh index 52e4f7a6..bc9f3258 100755 --- a/travis/mingw32/install.sh +++ b/travis/mingw32/install.sh @@ -2,6 +2,9 @@ set -e +unset CC +unset CXX + export CCACHE_CPP2=yes export PROJ_DB_CACHE_DIR="$HOME/.ccache" @@ -16,6 +19,22 @@ ln -s $MINGW_PREFIX/libgcc_s_seh-1.dll $WINE_SYSDIR ln -s $MINGW_PREFIX/libgcc_s_sjlj-1.dll $WINE_SYSDIR ln -s /usr/$MINGW_ARCH/lib/libwinpthread-1.dll $WINE_SYSDIR +# build zlib +wget https://github.com/madler/zlib/archive/v1.2.11.tar.gz +tar xzf v1.2.11.tar.gz +(cd zlib-1.2.11 && sudo make install -fwin32/Makefile.gcc SHARED_MODE=1 PREFIX=x86_64-w64-mingw32- DESTDIR=/usr/$MINGW_ARCH/) +sudo mkdir -p /usr/$MINGW_ARCH/include +sudo mkdir -p /usr/$MINGW_ARCH/lib +sudo cp /usr/$MINGW_ARCH/*.h /usr/$MINGW_ARCH/include +sudo cp /usr/$MINGW_ARCH/libz.* /usr/$MINGW_ARCH/lib +ln -s /usr/$MINGW_ARCH/zlib1.dll $WINE_SYSDIR + +# build libtiff +wget https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz +tar xzf tiff-4.1.0.tar.gz +(cd tiff-4.1.0 && ./configure --host=$MINGW_ARCH --prefix=/usr/$MINGW_ARCH && make -j2 && sudo make install) +ln -s /usr/$MINGW_ARCH/bin/libtiff-5.dll $WINE_SYSDIR + # build sqlite3 wget https://sqlite.org/2018/sqlite-autoconf-3250100.tar.gz tar xzf sqlite-autoconf-3250100.tar.gz diff --git a/travis/osx/before_install.sh b/travis/osx/before_install.sh index 09a7a74f..5de816c8 100755 --- a/travis/osx/before_install.sh +++ b/travis/osx/before_install.sh @@ -6,7 +6,8 @@ export PATH=$HOME/Library/Python/3.7/bin:$PATH brew update brew install ccache -brew install sqlite3 +#brew upgrade sqlite3 +#brew upgrade libtiff brew install doxygen brew install md5sha1sum brew reinstall python -- cgit v1.2.3 From 9f908ae47cfa70d3cdb2709a8ab5d8eeb10034fc Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 9 Dec 2019 23:26:39 +0100 Subject: Horizontal shift grids: hide the 'positive longitude shift value = westward correction' in the CTable2/NTv1/NTv2 backends --- src/apply_gridshift.cpp | 8 ++++---- src/grids.cpp | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp index a0ffa394..c786a50a 100644 --- a/src/apply_gridshift.cpp +++ b/src/apply_gridshift.cpp @@ -212,12 +212,12 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { return t; if (!inverse) { - in.lam -= t.lam; + in.lam += t.lam; in.phi += t.phi; return in; } - t.lam = tb.lam + t.lam; + t.lam = tb.lam - t.lam; t.phi = tb.phi - t.phi; do { @@ -235,8 +235,8 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { if (del.lam == HUGE_VAL) break; - dif.lam = t.lam - del.lam - tb.lam; - dif.phi = t.phi + del.phi - tb.phi; + dif.lam = t.lam + del.lam - tb.lam; + dif.phi = t.phi + del.phi - tb.phi; t.lam -= dif.lam; t.phi -= dif.phi; diff --git a/src/grids.cpp b/src/grids.cpp index 7d19b1f7..cc954542 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -469,7 +469,8 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { } /* convert seconds to radians */ latShift = static_cast(two_doubles[0] * ((M_PI / 180.0) / 3600.0)); - lonShift = static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = -static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); return true; } @@ -572,7 +573,8 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, } latShift = two_floats[1]; - lonShift = two_floats[0]; + // west longitude positive convention ! + lonShift = -two_floats[0]; return true; } @@ -642,7 +644,8 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { } /* convert seconds to radians */ latShift = static_cast(two_float[0] * ((M_PI / 180.0) / 3600.0)); - lonShift = static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = -static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); return true; } -- cgit v1.2.3 From 1f01ecb90a751c8099a50e066616639f63522134 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 6 Dec 2019 21:08:59 +0100 Subject: Add support for horizontal and vertical grids in GeoTIFF --- cmake/ProjTest.cmake | 19 +- data/Makefile.am | 38 +- data/tests/test_hgrid.tif | Bin 0 -> 506 bytes data/tests/test_hgrid_degree.tif | Bin 0 -> 680 bytes .../tests/test_hgrid_extra_ifd_with_other_info.tif | Bin 0 -> 1708 bytes data/tests/test_hgrid_lon_shift_first.tif | Bin 0 -> 764 bytes data/tests/test_hgrid_positive_west.tif | Bin 0 -> 764 bytes data/tests/test_hgrid_radian.tif | Bin 0 -> 680 bytes data/tests/test_hgrid_separate.tif | Bin 0 -> 514 bytes data/tests/test_hgrid_strip.tif | Bin 0 -> 514 bytes data/tests/test_hgrid_tiled.tif | Bin 0 -> 4920 bytes data/tests/test_hgrid_tiled_separate.tif | Bin 0 -> 2256 bytes data/tests/test_hgrid_with_overview.tif | Bin 0 -> 864 bytes data/tests/test_hgrid_with_subgrid.tif | Bin 0 -> 6943 bytes .../tests/test_hgrid_with_subgrid_no_grid_name.tif | Bin 0 -> 6943 bytes data/tests/test_vgrid_bigendian.tif | Bin 0 -> 430 bytes data/tests/test_vgrid_bigendian_bigtiff.tif | Bin 0 -> 568 bytes data/tests/test_vgrid_bigtiff.tif | Bin 0 -> 568 bytes data/tests/test_vgrid_bottomup_with_matrix.tif | Bin 0 -> 474 bytes data/tests/test_vgrid_bottomup_with_scale.tif | Bin 0 -> 430 bytes data/tests/test_vgrid_deflate.tif | Bin 0 -> 430 bytes .../test_vgrid_deflate_floatingpointpredictor.tif | Bin 0 -> 422 bytes data/tests/test_vgrid_float64.tif | Bin 0 -> 494 bytes data/tests/test_vgrid_in_second_channel.tif | Bin 0 -> 632 bytes data/tests/test_vgrid_int16.tif | Bin 0 -> 398 bytes data/tests/test_vgrid_int32.tif | Bin 0 -> 430 bytes data/tests/test_vgrid_invalid_channel_type.tif | Bin 0 -> 560 bytes data/tests/test_vgrid_nodata.tif | Bin 0 -> 464 bytes data/tests/test_vgrid_pixelisarea.tif | Bin 0 -> 430 bytes data/tests/test_vgrid_pixelispoint.tif | Bin 0 -> 430 bytes data/tests/test_vgrid_uint16.tif | Bin 0 -> 398 bytes data/tests/test_vgrid_uint16_with_scale_offset.tif | Bin 0 -> 556 bytes data/tests/test_vgrid_uint32.tif | Bin 0 -> 430 bytes data/tests/test_vgrid_unsupported_byte.tif | Bin 0 -> 382 bytes data/tests/test_vgrid_with_overview.tif | Bin 0 -> 707 bytes data/tests/test_vgrid_with_subgrid.tif | Bin 0 -> 756 bytes src/grids.cpp | 1230 ++++++++++++++++++++ src/grids.hpp | 3 + test/CMakeLists.txt | 1 + test/cli/test27 | 2 +- test/cli/test83 | 2 +- test/cli/testdatumfile | 6 +- test/gie/4D-API_cs2cs-style.gie | 2 - test/gie/Makefile.am | 8 +- test/gie/geotiff_grids.gie | 283 +++++ 45 files changed, 1582 insertions(+), 12 deletions(-) create mode 100644 data/tests/test_hgrid.tif create mode 100644 data/tests/test_hgrid_degree.tif create mode 100644 data/tests/test_hgrid_extra_ifd_with_other_info.tif create mode 100644 data/tests/test_hgrid_lon_shift_first.tif create mode 100644 data/tests/test_hgrid_positive_west.tif create mode 100644 data/tests/test_hgrid_radian.tif create mode 100644 data/tests/test_hgrid_separate.tif create mode 100644 data/tests/test_hgrid_strip.tif create mode 100644 data/tests/test_hgrid_tiled.tif create mode 100644 data/tests/test_hgrid_tiled_separate.tif create mode 100644 data/tests/test_hgrid_with_overview.tif create mode 100644 data/tests/test_hgrid_with_subgrid.tif create mode 100644 data/tests/test_hgrid_with_subgrid_no_grid_name.tif create mode 100644 data/tests/test_vgrid_bigendian.tif create mode 100644 data/tests/test_vgrid_bigendian_bigtiff.tif create mode 100644 data/tests/test_vgrid_bigtiff.tif create mode 100644 data/tests/test_vgrid_bottomup_with_matrix.tif create mode 100644 data/tests/test_vgrid_bottomup_with_scale.tif create mode 100644 data/tests/test_vgrid_deflate.tif create mode 100644 data/tests/test_vgrid_deflate_floatingpointpredictor.tif create mode 100644 data/tests/test_vgrid_float64.tif create mode 100644 data/tests/test_vgrid_in_second_channel.tif create mode 100644 data/tests/test_vgrid_int16.tif create mode 100644 data/tests/test_vgrid_int32.tif create mode 100644 data/tests/test_vgrid_invalid_channel_type.tif create mode 100644 data/tests/test_vgrid_nodata.tif create mode 100644 data/tests/test_vgrid_pixelisarea.tif create mode 100644 data/tests/test_vgrid_pixelispoint.tif create mode 100644 data/tests/test_vgrid_uint16.tif create mode 100644 data/tests/test_vgrid_uint16_with_scale_offset.tif create mode 100644 data/tests/test_vgrid_uint32.tif create mode 100644 data/tests/test_vgrid_unsupported_byte.tif create mode 100644 data/tests/test_vgrid_with_overview.tif create mode 100644 data/tests/test_vgrid_with_subgrid.tif create mode 100644 test/gie/geotiff_grids.gie diff --git a/cmake/ProjTest.cmake b/cmake/ProjTest.cmake index ad64d0ba..6abfbcd7 100644 --- a/cmake/ProjTest.cmake +++ b/cmake/ProjTest.cmake @@ -26,8 +26,13 @@ function(proj_add_test_script_sh SH_NAME BIN_USE) COMMAND ${PROJECT_SOURCE_DIR}/test/cli/${SH_NAME} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${BIN_USE}} ) - set_tests_properties( ${testname} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + if(MSVC) + set_tests_properties( ${testname} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + else() + set_tests_properties( ${testname} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + endif() endif() endif() @@ -43,8 +48,14 @@ function(proj_add_gie_test TESTNAME TESTCASE) COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${GIE_BIN} ${TESTFILE} ) - set_tests_properties( ${TESTNAME} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + + if(MSVC) + set_tests_properties( ${TESTNAME} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + else() + set_tests_properties( ${TESTNAME} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + endif() endfunction() diff --git a/data/Makefile.am b/data/Makefile.am index e7c38d7c..9a7c0c28 100644 --- a/data/Makefile.am +++ b/data/Makefile.am @@ -43,7 +43,43 @@ EXTRA_DIST = GL27 nad.lst nad27 nad83 \ CH \ ITRF2000 ITRF2008 ITRF2014 \ projjson.schema.json \ - CMakeLists.txt tests/test_nodata.gtx null \ + CMakeLists.txt \ + tests/test_nodata.gtx \ + tests/test_vgrid_bigendian_bigtiff.tif \ + tests/test_vgrid_bigendian.tif \ + tests/test_vgrid_bigtiff.tif \ + tests/test_vgrid_bottomup_with_matrix.tif \ + tests/test_vgrid_bottomup_with_scale.tif \ + tests/test_vgrid_deflate_floatingpointpredictor.tif \ + tests/test_vgrid_deflate.tif \ + tests/test_vgrid_float64.tif \ + tests/test_vgrid_in_second_channel.tif \ + tests/test_vgrid_int16.tif \ + tests/test_vgrid_int32.tif \ + tests/test_vgrid_uint32.tif \ + tests/test_vgrid_invalid_channel_type.tif \ + tests/test_vgrid_nodata.tif \ + tests/test_vgrid_pixelisarea.tif \ + tests/test_vgrid_pixelispoint.tif \ + tests/test_vgrid_uint16.tif \ + tests/test_vgrid_uint16_with_scale_offset.tif \ + tests/test_vgrid_unsupported_byte.tif \ + tests/test_vgrid_with_overview.tif \ + tests/test_vgrid_with_subgrid.tif \ + tests/test_hgrid.tif \ + tests/test_hgrid_separate.tif \ + tests/test_hgrid_tiled.tif \ + tests/test_hgrid_tiled_separate.tif \ + tests/test_hgrid_strip.tif \ + tests/test_hgrid_positive_west.tif \ + tests/test_hgrid_lon_shift_first.tif \ + tests/test_hgrid_radian.tif \ + tests/test_hgrid_degree.tif \ + tests/test_hgrid_with_overview.tif \ + tests/test_hgrid_extra_ifd_with_other_info.tif \ + tests/test_hgrid_with_subgrid.tif \ + tests/test_hgrid_with_subgrid_no_grid_name.tif \ + null \ generate_all_sql_in.cmake sql_filelist.cmake \ $(SQL_ORDERED_LIST) diff --git a/data/tests/test_hgrid.tif b/data/tests/test_hgrid.tif new file mode 100644 index 00000000..94718c21 Binary files /dev/null and b/data/tests/test_hgrid.tif differ diff --git a/data/tests/test_hgrid_degree.tif b/data/tests/test_hgrid_degree.tif new file mode 100644 index 00000000..d06782ec Binary files /dev/null and b/data/tests/test_hgrid_degree.tif differ diff --git a/data/tests/test_hgrid_extra_ifd_with_other_info.tif b/data/tests/test_hgrid_extra_ifd_with_other_info.tif new file mode 100644 index 00000000..b7e67af7 Binary files /dev/null and b/data/tests/test_hgrid_extra_ifd_with_other_info.tif differ diff --git a/data/tests/test_hgrid_lon_shift_first.tif b/data/tests/test_hgrid_lon_shift_first.tif new file mode 100644 index 00000000..395f0743 Binary files /dev/null and b/data/tests/test_hgrid_lon_shift_first.tif differ diff --git a/data/tests/test_hgrid_positive_west.tif b/data/tests/test_hgrid_positive_west.tif new file mode 100644 index 00000000..4ebc17cc Binary files /dev/null and b/data/tests/test_hgrid_positive_west.tif differ diff --git a/data/tests/test_hgrid_radian.tif b/data/tests/test_hgrid_radian.tif new file mode 100644 index 00000000..30219ccd Binary files /dev/null and b/data/tests/test_hgrid_radian.tif differ diff --git a/data/tests/test_hgrid_separate.tif b/data/tests/test_hgrid_separate.tif new file mode 100644 index 00000000..ef2ca575 Binary files /dev/null and b/data/tests/test_hgrid_separate.tif differ diff --git a/data/tests/test_hgrid_strip.tif b/data/tests/test_hgrid_strip.tif new file mode 100644 index 00000000..e38fc609 Binary files /dev/null and b/data/tests/test_hgrid_strip.tif differ diff --git a/data/tests/test_hgrid_tiled.tif b/data/tests/test_hgrid_tiled.tif new file mode 100644 index 00000000..b0d5dd8b Binary files /dev/null and b/data/tests/test_hgrid_tiled.tif differ diff --git a/data/tests/test_hgrid_tiled_separate.tif b/data/tests/test_hgrid_tiled_separate.tif new file mode 100644 index 00000000..d7e0934f Binary files /dev/null and b/data/tests/test_hgrid_tiled_separate.tif differ diff --git a/data/tests/test_hgrid_with_overview.tif b/data/tests/test_hgrid_with_overview.tif new file mode 100644 index 00000000..d7453b49 Binary files /dev/null and b/data/tests/test_hgrid_with_overview.tif differ diff --git a/data/tests/test_hgrid_with_subgrid.tif b/data/tests/test_hgrid_with_subgrid.tif new file mode 100644 index 00000000..2acc4581 Binary files /dev/null and b/data/tests/test_hgrid_with_subgrid.tif differ diff --git a/data/tests/test_hgrid_with_subgrid_no_grid_name.tif b/data/tests/test_hgrid_with_subgrid_no_grid_name.tif new file mode 100644 index 00000000..974699b5 Binary files /dev/null and b/data/tests/test_hgrid_with_subgrid_no_grid_name.tif differ diff --git a/data/tests/test_vgrid_bigendian.tif b/data/tests/test_vgrid_bigendian.tif new file mode 100644 index 00000000..5cf4a039 Binary files /dev/null and b/data/tests/test_vgrid_bigendian.tif differ diff --git a/data/tests/test_vgrid_bigendian_bigtiff.tif b/data/tests/test_vgrid_bigendian_bigtiff.tif new file mode 100644 index 00000000..a586b85f Binary files /dev/null and b/data/tests/test_vgrid_bigendian_bigtiff.tif differ diff --git a/data/tests/test_vgrid_bigtiff.tif b/data/tests/test_vgrid_bigtiff.tif new file mode 100644 index 00000000..2a01893a Binary files /dev/null and b/data/tests/test_vgrid_bigtiff.tif differ diff --git a/data/tests/test_vgrid_bottomup_with_matrix.tif b/data/tests/test_vgrid_bottomup_with_matrix.tif new file mode 100644 index 00000000..90f637dc Binary files /dev/null and b/data/tests/test_vgrid_bottomup_with_matrix.tif differ diff --git a/data/tests/test_vgrid_bottomup_with_scale.tif b/data/tests/test_vgrid_bottomup_with_scale.tif new file mode 100644 index 00000000..636b7dc7 Binary files /dev/null and b/data/tests/test_vgrid_bottomup_with_scale.tif differ diff --git a/data/tests/test_vgrid_deflate.tif b/data/tests/test_vgrid_deflate.tif new file mode 100644 index 00000000..ee3b5f0a Binary files /dev/null and b/data/tests/test_vgrid_deflate.tif differ diff --git a/data/tests/test_vgrid_deflate_floatingpointpredictor.tif b/data/tests/test_vgrid_deflate_floatingpointpredictor.tif new file mode 100644 index 00000000..5fd7b9fa Binary files /dev/null and b/data/tests/test_vgrid_deflate_floatingpointpredictor.tif differ diff --git a/data/tests/test_vgrid_float64.tif b/data/tests/test_vgrid_float64.tif new file mode 100644 index 00000000..16b3e790 Binary files /dev/null and b/data/tests/test_vgrid_float64.tif differ diff --git a/data/tests/test_vgrid_in_second_channel.tif b/data/tests/test_vgrid_in_second_channel.tif new file mode 100644 index 00000000..d377f8b7 Binary files /dev/null and b/data/tests/test_vgrid_in_second_channel.tif differ diff --git a/data/tests/test_vgrid_int16.tif b/data/tests/test_vgrid_int16.tif new file mode 100644 index 00000000..1c69b5d6 Binary files /dev/null and b/data/tests/test_vgrid_int16.tif differ diff --git a/data/tests/test_vgrid_int32.tif b/data/tests/test_vgrid_int32.tif new file mode 100644 index 00000000..1b6dfd7b Binary files /dev/null and b/data/tests/test_vgrid_int32.tif differ diff --git a/data/tests/test_vgrid_invalid_channel_type.tif b/data/tests/test_vgrid_invalid_channel_type.tif new file mode 100644 index 00000000..ec9e641f Binary files /dev/null and b/data/tests/test_vgrid_invalid_channel_type.tif differ diff --git a/data/tests/test_vgrid_nodata.tif b/data/tests/test_vgrid_nodata.tif new file mode 100644 index 00000000..65ec5343 Binary files /dev/null and b/data/tests/test_vgrid_nodata.tif differ diff --git a/data/tests/test_vgrid_pixelisarea.tif b/data/tests/test_vgrid_pixelisarea.tif new file mode 100644 index 00000000..a5409f66 Binary files /dev/null and b/data/tests/test_vgrid_pixelisarea.tif differ diff --git a/data/tests/test_vgrid_pixelispoint.tif b/data/tests/test_vgrid_pixelispoint.tif new file mode 100644 index 00000000..cfeb598f Binary files /dev/null and b/data/tests/test_vgrid_pixelispoint.tif differ diff --git a/data/tests/test_vgrid_uint16.tif b/data/tests/test_vgrid_uint16.tif new file mode 100644 index 00000000..a03d9a73 Binary files /dev/null and b/data/tests/test_vgrid_uint16.tif differ diff --git a/data/tests/test_vgrid_uint16_with_scale_offset.tif b/data/tests/test_vgrid_uint16_with_scale_offset.tif new file mode 100644 index 00000000..b08fa4a3 Binary files /dev/null and b/data/tests/test_vgrid_uint16_with_scale_offset.tif differ diff --git a/data/tests/test_vgrid_uint32.tif b/data/tests/test_vgrid_uint32.tif new file mode 100644 index 00000000..cae7e9e7 Binary files /dev/null and b/data/tests/test_vgrid_uint32.tif differ diff --git a/data/tests/test_vgrid_unsupported_byte.tif b/data/tests/test_vgrid_unsupported_byte.tif new file mode 100644 index 00000000..ccf03fc8 Binary files /dev/null and b/data/tests/test_vgrid_unsupported_byte.tif differ diff --git a/data/tests/test_vgrid_with_overview.tif b/data/tests/test_vgrid_with_overview.tif new file mode 100644 index 00000000..aa15aa1d Binary files /dev/null and b/data/tests/test_vgrid_with_overview.tif differ diff --git a/data/tests/test_vgrid_with_subgrid.tif b/data/tests/test_vgrid_with_subgrid.tif new file mode 100644 index 00000000..5c7584c4 Binary files /dev/null and b/data/tests/test_vgrid_with_subgrid.tif differ diff --git a/src/grids.cpp b/src/grids.cpp index cc954542..010ddb22 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -34,6 +34,11 @@ #include "proj/internal/internal.hpp" #include "proj_internal.h" +#include "tiffio.h" + +#include +#include + NS_PROJ_START using namespace internal; @@ -64,12 +69,28 @@ static void swap_words(void *dataIn, size_t word_size, size_t word_count) } } +// --------------------------------------------------------------------------- + bool ExtentAndRes::fullWorldLongitude() const { return eastLon - westLon + resLon >= 2 * M_PI - 1e-10; } // --------------------------------------------------------------------------- +bool ExtentAndRes::contains(const ExtentAndRes &other) const { + return other.westLon >= westLon && other.eastLon <= eastLon && + other.southLat >= southLat && other.northLat <= northLat; +} + +// --------------------------------------------------------------------------- + +bool ExtentAndRes::intersects(const ExtentAndRes &other) const { + return other.westLon < eastLon && westLon <= other.westLon && + other.southLat < northLat && southLat <= other.northLat; +} + +// --------------------------------------------------------------------------- + Grid::Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn) : m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} @@ -246,6 +267,926 @@ VerticalShiftGridSet::~VerticalShiftGridSet() = default; // --------------------------------------------------------------------------- +static bool IsTIFF(size_t header_size, const unsigned char *header) { + // Test combinations of signature for ClassicTIFF/BigTIFF little/big endian + return header_size >= 4 && (((header[0] == 'I' && header[1] == 'I') || + (header[0] == 'M' && header[1] == 'M')) && + ((header[2] == 0x2A && header[3] == 0) || + (header[3] == 0x2A && header[2] == 0) || + (header[2] == 0x2B && header[3] == 0) || + (header[3] == 0x2B && header[2] == 0))); +} + +// --------------------------------------------------------------------------- + +enum class TIFFDataType { Int16, UInt16, Int32, UInt32, Float32, Float64 }; + +// --------------------------------------------------------------------------- + +constexpr uint16 TIFFTAG_GEOPIXELSCALE = 33550; +constexpr uint16 TIFFTAG_GEOTIEPOINTS = 33922; +constexpr uint16 TIFFTAG_GEOTRANSMATRIX = 34264; +constexpr uint16 TIFFTAG_GEOKEYDIRECTORY = 34735; +constexpr uint16 TIFFTAG_GEODOUBLEPARAMS = 34736; +constexpr uint16 TIFFTAG_GEOASCIIPARAMS = 34737; +constexpr uint16 TIFFTAG_GDAL_METADATA = 42112; +constexpr uint16 TIFFTAG_GDAL_NODATA = 42113; + +// --------------------------------------------------------------------------- + +class GTiffGrid : public Grid { + PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset + TIFF *m_hTIFF; // owned by the belonging GTiffDataset + TIFFDataType m_dt; + uint16 m_samplesPerPixel; + uint16 m_planarConfig; + bool m_bottomUp; + toff_t m_dirOffset; + bool m_tiled; + uint32 m_blockWidth = 0; + uint32 m_blockHeight = 0; + mutable int m_lastSample = -1; + mutable int m_lastBlockX = -1; + mutable int m_lastBlockY = -1; + mutable std::vector m_buffer{}; + unsigned m_blocksPerRow = 0; + unsigned m_blocksPerCol = 0; + std::map m_mapOffset{}; + std::map m_mapScale{}; + std::map, std::string> m_metadata{}; + bool m_hasNodata = false; + float m_noData = 0.0f; + uint32 m_subfileType = 0; + + GTiffGrid(const GTiffGrid &) = delete; + GTiffGrid &operator=(const GTiffGrid &) = delete; + + void getScaleOffset(double &scale, double &offset, uint16 sample) const; + + template + float readValue(uint32 offsetInBlock, uint16 sample) const; + + public: + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, + const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); + + ~GTiffGrid() override; + + uint16 samplesPerPixel() const { return m_samplesPerPixel; } + + bool valueAt(uint16 sample, int x, int y, float &out) const; + + bool isNodata(float val) const; + + std::string metadataItem(const std::string &key, int sample = -1) const; + + uint32 subfileType() const { return m_subfileType; } +}; + +// --------------------------------------------------------------------------- + +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, + const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, + bool bottomUpIn) + : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), m_dt(dtIn), + m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), + m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), + m_tiled(TIFFIsTiled(hTIFF) != 0) { + + if (m_tiled) { + TIFFGetField(m_hTIFF, TIFFTAG_TILEWIDTH, &m_blockWidth); + TIFFGetField(m_hTIFF, TIFFTAG_TILELENGTH, &m_blockHeight); + } else { + m_blockWidth = widthIn; + TIFFGetField(m_hTIFF, TIFFTAG_ROWSPERSTRIP, &m_blockHeight); + if (m_blockHeight > static_cast(m_height)) + m_blockHeight = m_height; + } + + TIFFGetField(m_hTIFF, TIFFTAG_SUBFILETYPE, &m_subfileType); + + m_blocksPerRow = (m_width + m_blockWidth - 1) / m_blockWidth; + m_blocksPerCol = (m_height + m_blockHeight - 1) / m_blockHeight; + + const char *text = nullptr; + // Poor-man XML parsing of TIFFTAG_GDAL_METADATA tag. Hopefully good + // enough for our purposes. + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_METADATA, &text)) { + const char *ptr = text; + while (true) { + ptr = strstr(ptr, "'); + if (endTag == nullptr) + break; + const char *endValue = strchr(endTag, '<'); + if (endValue == nullptr) + break; + + std::string tag; + tag.append(ptr, endTag - ptr); + + std::string value; + value.append(endTag + 1, endValue - (endTag + 1)); + + std::string name; + auto namePos = tag.find("name=\""); + if (namePos == std::string::npos) + break; + { + namePos += strlen("name=\""); + const auto endQuote = tag.find('"', namePos); + if (endQuote == std::string::npos) + break; + name = tag.substr(namePos, endQuote - namePos); + } + + const auto samplePos = tag.find("sample=\""); + int sample = -1; + if (samplePos != std::string::npos) { + sample = atoi(tag.c_str() + samplePos + strlen("sample=\"")); + } + + m_metadata[std::pair(sample, name)] = value; + + auto rolePos = tag.find("role=\""); + if (rolePos != std::string::npos) { + rolePos += strlen("role=\""); + const auto endQuote = tag.find('"', rolePos); + if (endQuote == std::string::npos) + break; + const auto role = tag.substr(rolePos, endQuote - rolePos); + if (role == "offset") { + if (sample >= 0) { + try { + m_mapOffset[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } else if (role == "scale") { + if (sample >= 0) { + try { + m_mapScale[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } + } + + ptr = endValue + 1; + } + } + + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_NODATA, &text)) { + try { + m_noData = static_cast(c_locale_stod(text)); + m_hasNodata = true; + } catch (const std::exception &) { + } + } +} + +// --------------------------------------------------------------------------- + +GTiffGrid::~GTiffGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffGrid::getScaleOffset(double &scale, double &offset, + uint16 sample) const { + { + auto iter = m_mapScale.find(sample); + if (iter != m_mapScale.end()) + scale = iter->second; + } + + { + auto iter = m_mapOffset.find(sample); + if (iter != m_mapOffset.end()) + offset = iter->second; + } +} + +// --------------------------------------------------------------------------- + +template +float GTiffGrid::readValue(uint32 offsetInBlock, uint16 sample) const { + const auto ptr = reinterpret_cast(m_buffer.data()); + assert(offsetInBlock < m_buffer.size() / sizeof(T)); + const auto val = ptr[offsetInBlock]; + if (!m_hasNodata || static_cast(val) != m_noData) { + double scale = 1; + double offset = 0; + getScaleOffset(scale, offset, sample); + return static_cast(val * scale + offset); + } else { + return static_cast(val); + } +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, + float &out) const { + assert(x >= 0 && yFromBottom >= 0 && x < m_width && yFromBottom < m_height); + assert(sample < m_samplesPerPixel); + if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && + !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { + return false; + } + if (m_buffer.empty()) { + const auto blockSize = static_cast( + m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); + try { + m_buffer.resize(blockSize); + } catch (const std::exception &e) { + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + return false; + } + } + + const int blockX = x / m_blockWidth; + + // All non-TIFF grids have the first rows in the file being the one + // corresponding to the southern-most row. In GeoTIFF, the convention is + // *generally* different (when m_bottomUp == false), TIFF being an + // image-oriented image. If m_bottomUp == true, then we had GeoTIFF hints + // that the first row of the image is the southern-most. + const int yTIFF = m_bottomUp ? yFromBottom : m_height - 1 - yFromBottom; + const int blockY = yTIFF / m_blockHeight; + + if (m_lastSample != static_cast(sample) || m_lastBlockX != blockX || + m_lastBlockY != blockY) { + uint32 blockId = blockY * m_blocksPerRow + blockX; + if (m_planarConfig == PLANARCONFIG_SEPARATE) { + blockId += sample * m_blocksPerCol * m_blocksPerRow; + } + if (m_tiled) { + if (TIFFReadEncodedTile(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } else { + if (TIFFReadEncodedStrip(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } + m_lastSample = sample; + m_lastBlockX = blockX; + m_lastBlockY = blockY; + } + uint32 offsetInBlock = + (x % m_blockWidth) + (yTIFF % m_blockHeight) * m_blockWidth; + if (m_planarConfig == PLANARCONFIG_CONTIG) + offsetInBlock = offsetInBlock * m_samplesPerPixel + sample; + + switch (m_dt) { + case TIFFDataType::Int16: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::UInt16: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::Int32: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::UInt32: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::Float32: + out = readValue(offsetInBlock, sample); + break; + + case TIFFDataType::Float64: + out = readValue(offsetInBlock, sample); + break; + } + + return true; +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::isNodata(float val) const { + return (m_hasNodata && val == m_noData) || std::isnan(val); +} + +// --------------------------------------------------------------------------- + +std::string GTiffGrid::metadataItem(const std::string &key, int sample) const { + auto iter = m_metadata.find(std::pair(sample, key)); + if (iter == m_metadata.end()) { + return std::string(); + } + return iter->second; +} + +// --------------------------------------------------------------------------- + +class GTiffDataset { + PJ_CONTEXT *m_ctx; + PAFile m_fp; + TIFF *m_hTIFF = nullptr; + bool m_hasNextGrid = false; + toff_t m_nextDirOffset = 0; + std::string m_filename{}; + + GTiffDataset(const GTiffDataset &) = delete; + GTiffDataset &operator=(const GTiffDataset &) = delete; + + // libtiff I/O routines + static tsize_t tiffReadProc(thandle_t fd, tdata_t buf, tsize_t size) { + GTiffDataset *self = static_cast(fd); + return pj_ctx_fread(self->m_ctx, buf, 1, size, self->m_fp); + } + + static tsize_t tiffWriteProc(thandle_t, tdata_t, tsize_t) { + assert(false); + return 0; + } + + static toff_t tiffSeekProc(thandle_t fd, toff_t off, int whence) { + GTiffDataset *self = static_cast(fd); + // FIXME Remove cast to long when pj_ctx_fseek supports unsigned long + // long + if (pj_ctx_fseek(self->m_ctx, self->m_fp, static_cast(off), + whence) == 0) + return static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); + else + return static_cast(-1); + } + + static int tiffCloseProc(thandle_t) { + // done in destructor + return 0; + } + + static toff_t tiffSizeProc(thandle_t fd) { + GTiffDataset *self = static_cast(fd); + const auto old_off = pj_ctx_ftell(self->m_ctx, self->m_fp); + pj_ctx_fseek(self->m_ctx, self->m_fp, 0, SEEK_END); + const auto file_size = + static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); + pj_ctx_fseek(self->m_ctx, self->m_fp, old_off, SEEK_SET); + return file_size; + } + + static int tiffMapProc(thandle_t, tdata_t *, toff_t *) { return (0); } + + static void tiffUnmapProc(thandle_t, tdata_t, toff_t) {} + + public: + GTiffDataset(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + virtual ~GTiffDataset(); + + bool openTIFF(const std::string &filename); + + std::unique_ptr nextGrid(); +}; + +// --------------------------------------------------------------------------- + +GTiffDataset::~GTiffDataset() { + if (m_hTIFF) + TIFFClose(m_hTIFF); + pj_ctx_fclose(m_ctx, m_fp); +} + +// --------------------------------------------------------------------------- +class OneTimeTIFFTagInit { + + static TIFFExtendProc ParentExtender; + + // Function called by libtiff when initializing a TIFF directory + static void GTiffTagExtender(TIFF *tif) { + static const TIFFFieldInfo xtiffFieldInfo[] = { + // GeoTIFF tags + {TIFFTAG_GEOPIXELSCALE, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoPixelScale")}, + {TIFFTAG_GEOTIEPOINTS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoTiePoints")}, + {TIFFTAG_GEOTRANSMATRIX, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoTransformationMatrix")}, + + {TIFFTAG_GEOKEYDIRECTORY, -1, -1, TIFF_SHORT, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoKeyDirectory")}, + {TIFFTAG_GEODOUBLEPARAMS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast("GeoDoubleParams")}, + {TIFFTAG_GEOASCIIPARAMS, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast("GeoASCIIParams")}, + + // GDAL tags + {TIFFTAG_GDAL_METADATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast("GDALMetadata")}, + {TIFFTAG_GDAL_NODATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, FALSE, + const_cast("GDALNoDataValue")}, + + }; + + if (ParentExtender) + (*ParentExtender)(tif); + + TIFFMergeFieldInfo(tif, xtiffFieldInfo, + sizeof(xtiffFieldInfo) / sizeof(xtiffFieldInfo[0])); + } + + public: + OneTimeTIFFTagInit() { + assert(ParentExtender == nullptr); + // Install our TIFF tag extender + ParentExtender = TIFFSetTagExtender(GTiffTagExtender); + } +}; + +TIFFExtendProc OneTimeTIFFTagInit::ParentExtender = nullptr; + +// --------------------------------------------------------------------------- + +bool GTiffDataset::openTIFF(const std::string &filename) { + static OneTimeTIFFTagInit oneTimeTIFFTagInit; + m_hTIFF = + TIFFClientOpen(filename.c_str(), "r", static_cast(this), + GTiffDataset::tiffReadProc, GTiffDataset::tiffWriteProc, + GTiffDataset::tiffSeekProc, GTiffDataset::tiffCloseProc, + GTiffDataset::tiffSizeProc, GTiffDataset::tiffMapProc, + GTiffDataset::tiffUnmapProc); + + m_filename = filename; + m_hasNextGrid = true; + return m_hTIFF != nullptr; +} +// --------------------------------------------------------------------------- + +std::unique_ptr GTiffDataset::nextGrid() { + if (!m_hasNextGrid) + return nullptr; + if (m_nextDirOffset) { + TIFFSetSubDirectory(m_hTIFF, m_nextDirOffset); + } + + uint32 width = 0; + uint32 height = 0; + TIFFGetField(m_hTIFF, TIFFTAG_IMAGEWIDTH, &width); + TIFFGetField(m_hTIFF, TIFFTAG_IMAGELENGTH, &height); + if (width == 0 || height == 0 || width > INT_MAX || height > INT_MAX) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid image size"); + return nullptr; + } + + uint16 samplesPerPixel = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLESPERPIXEL, &samplesPerPixel)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SamplesPerPixel tag"); + return nullptr; + } + if (samplesPerPixel == 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid SamplesPerPixel value"); + return nullptr; + } + + uint16 bitsPerSample = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_BITSPERSAMPLE, &bitsPerSample)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing BitsPerSample tag"); + return nullptr; + } + + uint16 planarConfig = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PLANARCONFIG, &planarConfig)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing PlanarConfig tag"); + return nullptr; + } + + uint16 sampleFormat = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLEFORMAT, &sampleFormat)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SampleFormat tag"); + return nullptr; + } + + TIFFDataType dt; + if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 16) + dt = TIFFDataType::Int16; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 16) + dt = TIFFDataType::UInt16; + else if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 32) + dt = TIFFDataType::Int32; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 32) + dt = TIFFDataType::UInt32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 32) + dt = TIFFDataType::Float32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 64) + dt = TIFFDataType::Float64; + else { + pj_log( + m_ctx, PJ_LOG_ERROR, + "Unsupported combination of SampleFormat and BitsPerSample values"); + return nullptr; + } + + uint16 photometric = PHOTOMETRIC_MINISBLACK; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PHOTOMETRIC, &photometric)) + photometric = PHOTOMETRIC_MINISBLACK; + if (photometric != PHOTOMETRIC_MINISBLACK) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported Photometric value"); + return nullptr; + } + + uint16 compression = COMPRESSION_NONE; + if (!TIFFGetField(m_hTIFF, TIFFTAG_COMPRESSION, &compression)) + compression = COMPRESSION_NONE; + + if (compression != COMPRESSION_NONE && + !TIFFIsCODECConfigured(compression)) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Cannot open TIFF file due to missing codec."); + return nullptr; + } + // We really don't want to try dealing with old-JPEG images + if (compression == COMPRESSION_OJPEG) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported compression method."); + return nullptr; + } + + const auto blockSize = TIFFIsTiled(m_hTIFF) ? TIFFTileSize64(m_hTIFF) + : TIFFStripSize64(m_hTIFF); + if (blockSize == 0 || blockSize > 64 * 1024 * 2014) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported block size."); + return nullptr; + } + + unsigned short count = 0; + unsigned short *geokeys = nullptr; + bool pixelIsArea = false; + if (!TIFFGetField(m_hTIFF, TIFFTAG_GEOKEYDIRECTORY, &count, &geokeys)) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, "No GeoKeys tag"); + } else { + if (count < 4 || (count % 4) != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoKeys tag"); + return nullptr; + } + + if (geokeys[0] != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported GeoTIFF major version"); + return nullptr; + } + // We only know that we support GeoTIFF 1.0 and 1.1 at that time + if (geokeys[1] != 1 || geokeys[2] > 1) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, + "GeoTIFF %d.%d possibly not handled", geokeys[1], + geokeys[2]); + } + + for (unsigned int i = 4; i + 3 < count; i += 4) { + constexpr unsigned short GTModelTypeGeoKey = 1024; + constexpr unsigned short ModelTypeGeographic = 2; + + constexpr unsigned short GTRasterTypeGeoKey = 1025; + constexpr unsigned short RasterPixelIsArea = 1; + // constexpr unsigned short RasterPixelIsPoint = 2; + + if (geokeys[i] == GTModelTypeGeoKey) { + if (geokeys[i + 3] != ModelTypeGeographic) { + pj_log(m_ctx, PJ_LOG_ERROR, "Only GTModelTypeGeoKey = " + "ModelTypeGeographic is " + "supported"); + return nullptr; + } + } else if (geokeys[i] == GTRasterTypeGeoKey) { + if (geokeys[i + 3] == RasterPixelIsArea) { + pixelIsArea = true; + } + } + } + } + + double hRes = 0; + double vRes = 0; + double westLon = 0; + double northLat = 0; + + double *matrix = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTRANSMATRIX, &count, &matrix) && + count == 16) { + // If using GDAL to produce a bottom-up georeferencing, it will produce + // a GeoTransformationMatrix, since negative values in GeoPixelScale + // have historically been implementation bugs. + if (matrix[1] != 0 || matrix[4] != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Rotational terms not supported in " + "GeoTransformationMatrix tag"); + return nullptr; + } + + westLon = matrix[3]; + hRes = matrix[0]; + northLat = matrix[7]; + vRes = -matrix[5]; // negation to simulate GeoPixelScale convention + } else { + double *geopixelscale = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOPIXELSCALE, &count, + &geopixelscale) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoPixelScale tag"); + return nullptr; + } + if (count != 3) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoPixelScale tag"); + return nullptr; + } + hRes = geopixelscale[0]; + vRes = geopixelscale[1]; + + double *geotiepoints = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTIEPOINTS, &count, + &geotiepoints) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoTiePoints tag"); + return nullptr; + } + if (count != 6) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoTiePoints tag"); + return nullptr; + } + + westLon = geotiepoints[3] - geotiepoints[0] * hRes; + northLat = geotiepoints[4] + geotiepoints[1] * vRes; + } + + if (pixelIsArea) { + westLon += 0.5 * hRes; + northLat -= 0.5 * vRes; + } + + ExtentAndRes extent; + extent.westLon = westLon * DEG_TO_RAD; + extent.northLat = northLat * DEG_TO_RAD; + extent.resLon = hRes * DEG_TO_RAD; + extent.resLat = fabs(vRes) * DEG_TO_RAD; + extent.eastLon = (westLon + hRes * (width - 1)) * DEG_TO_RAD; + extent.southLat = (northLat - vRes * (height - 1)) * DEG_TO_RAD; + + if (vRes < 0) { + std::swap(extent.northLat, extent.southLat); + } + + if (!(fabs(extent.westLon) <= 4 * M_PI && + fabs(extent.eastLon) <= 4 * M_PI && + fabs(extent.northLat) <= M_PI + 1e-5 && + fabs(extent.southLat) <= M_PI + 1e-5 && + extent.westLon < extent.eastLon && + extent.southLat < extent.northLat && extent.resLon > 1e-10 && + extent.resLat > 1e-10)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Inconsistent georeferencing for %s", + m_filename.c_str()); + return nullptr; + } + + auto ret = std::unique_ptr( + new GTiffGrid(m_ctx, m_hTIFF, width, height, extent, dt, + samplesPerPixel, planarConfig, vRes < 0)); + m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; + m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); + return ret; +} + +// --------------------------------------------------------------------------- + +class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { + + GTiffVGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + + public: + ~GTiffVGridShiftSet() override; + + static std::unique_ptr + open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); +}; + +// --------------------------------------------------------------------------- + +template +static void +insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids) { + const auto &extent = grid->extentAndRes(); + + // If we have one or both of grid_name and parent_name, try to use + // the names to recreate the hiearchy + if (!gridName.empty()) { + if (mapGrids.find(gridName) != mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Several grids called %s found!", + gridName.c_str()); + } + mapGrids[gridName] = grid.get(); + } + bool gridInserted = false; + if (!parentName.empty()) { + auto iter = mapGrids.find(parentName); + if (iter == mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to non-existing parent %s. " + "Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } else { + if (iter->second->extentAndRes().contains(extent)) { + iter->second->m_children.emplace_back(std::move(grid)); + gridInserted = true; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to parent %s, but its extent is " + "not included in it. Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } + } + } else if (!gridName.empty()) { + topGrids.emplace_back(std::move(grid)); + gridInserted = true; + } + + // Fallback to analyzing spatial extents + if (!gridInserted) { + for (const auto &candidateParent : topGrids) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(grid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + topGrids.emplace_back(std::move(grid)); + } + } +} + +// --------------------------------------------------------------------------- + +class GTiffVGrid : public VerticalShiftGrid { + friend void insertIntoHierarchy( + PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids); + + std::unique_ptr m_grid; + uint16 m_idxSample; + + public: + GTiffVGrid(std::unique_ptr &&grid, uint16 idxSample); + + ~GTiffVGrid() override; + + bool valueAt(int x, int y, float &out) const override { + return m_grid->valueAt(m_idxSample, x, y, out); + } + + bool isNodata(float val, double /* multiplier */) const override { + return m_grid->isNodata(val); + } + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); +}; + +// --------------------------------------------------------------------------- + +GTiffVGridShiftSet::~GTiffVGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffVGrid::GTiffVGrid(std::unique_ptr &&grid, uint16 idxSample) + : VerticalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + m_grid(std::move(grid)), m_idxSample(idxSample) {} + +// --------------------------------------------------------------------------- + +GTiffVGrid::~GTiffVGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffVGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr +GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &filename) { + auto set = + std::unique_ptr(new GTiffVGridShiftSet(ctx, fp)); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + uint16 idxSample = 0; + + std::map mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + // Identify the index of the geoid_undulation/vertical_offset + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForShift = false; + for (int i = 0; i < static_cast(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "geoid_undulation" || desc == "vertical_offset") { + idxSample = static_cast(i); + foundDescriptionForShift = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForShift) { + if (ifd > 0) { + // Assuming that extra IFD without our channel of interest + // can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "geoid_undulation/vertical_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "geoid_undulation/vertical_offset channel"); + return nullptr; + } + } + } + + if (idxSample >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_name"); + + auto vgrid = + internal::make_unique(std::move(grid), idxSample); + + insertIntoHierarchy(ctx, std::move(vgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} + +// --------------------------------------------------------------------------- + std::unique_ptr VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (filename == "null") { @@ -277,6 +1218,25 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size; + if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != + sizeof(header)) { + pj_ctx_fclose(ctx, fp); + return nullptr; + } + pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + + if (IsTIFF(header_size, header)) { + auto set = GTiffVGridShiftSet::open(ctx, fp, filename); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; + } + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); pj_ctx_fclose(ctx, fp); return nullptr; @@ -782,6 +1742,270 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, // --------------------------------------------------------------------------- +class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { + + GTiffHGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + + public: + ~GTiffHGridShiftSet() override; + + static std::unique_ptr + open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); +}; + +// --------------------------------------------------------------------------- + +class GTiffHGrid : public HorizontalShiftGrid { + friend void insertIntoHierarchy( + PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids); + + std::unique_ptr m_grid; + uint16 m_idxLatShift; + uint16 m_idxLonShift; + double m_convFactorToRadian; + bool m_positiveEast; + + public: + GTiffHGrid(std::unique_ptr &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast); + + ~GTiffHGrid() override; + + bool valueAt(int x, int y, float &lonShift, float &latShift) const override; + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); +}; + +// --------------------------------------------------------------------------- + +GTiffHGridShiftSet::~GTiffHGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffHGrid::GTiffHGrid(std::unique_ptr &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast) + : HorizontalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + m_grid(std::move(grid)), m_idxLatShift(idxLatShift), + m_idxLonShift(idxLonShift), m_convFactorToRadian(convFactorToRadian), + m_positiveEast(positiveEast) {} + +// --------------------------------------------------------------------------- + +GTiffHGrid::~GTiffHGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffHGrid::valueAt(int x, int y, float &lonShift, float &latShift) const { + if (!m_grid->valueAt(m_idxLatShift, x, y, latShift) || + !m_grid->valueAt(m_idxLonShift, x, y, lonShift)) { + return false; + } + // From arc-seconds to radians + latShift = static_cast(latShift * m_convFactorToRadian); + lonShift = static_cast(lonShift * m_convFactorToRadian); + if (!m_positiveEast) { + lonShift = -lonShift; + } + return true; +} + +// --------------------------------------------------------------------------- + +void GTiffHGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr +GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &filename) { + auto set = + std::unique_ptr(new GTiffHGridShiftSet(ctx, fp)); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + + // Defaults inspired from NTv2 + uint16 idxLatShift = 0; + uint16 idxLonShift = 1; + constexpr double ARC_SECOND_TO_RADIAN = (M_PI / 180.0) / 3600.0; + double convFactorToRadian = ARC_SECOND_TO_RADIAN; + bool positiveEast = true; + + std::map mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + if (grid->samplesPerPixel() < 2) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, + "At least 2 samples per pixel needed"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has not at least 2 samples", ifd); + continue; + } + } + + // Identify the index of the latitude and longitude offset channels + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForLatOffset = false; + bool foundDescriptionForLonOffset = false; + for (int i = 0; i < static_cast(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "latitude_offset") { + idxLatShift = static_cast(i); + foundDescriptionForLatOffset = true; + } else if (desc == "longitude_offset") { + idxLonShift = static_cast(i); + foundDescriptionForLonOffset = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + if (ifd > 0) { + // Assuming that extra IFD without + // longitude_offset/latitude_offset can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "longitude_offset/latitude_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "longitude_offset/latitude_offset channel"); + return nullptr; + } + } + } + if (foundDescriptionForLatOffset && !foundDescriptionForLonOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found latitude_offset channel, but not longitude_offset"); + return nullptr; + } else if (foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found longitude_offset channel, but not latitude_offset"); + return nullptr; + } + + if (idxLatShift >= grid->samplesPerPixel() || + idxLonShift >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + if (foundDescriptionForLonOffset) { + const std::string positiveValue = + grid->metadataItem("positive_value", idxLonShift); + if (!positiveValue.empty()) { + if (positiveValue == "west") { + positiveEast = false; + } else if (positiveValue == "east") { + positiveEast = true; + } else { + pj_log(ctx, PJ_LOG_ERROR, + "Unsupported value %s for 'positive_value'", + positiveValue.c_str()); + return nullptr; + } + } + } + + // Identify their unit + { + const auto unitLatShift = + grid->metadataItem("UNITTYPE", idxLatShift); + const auto unitLonShift = + grid->metadataItem("UNITTYPE", idxLonShift); + if (unitLatShift != unitLonShift) { + pj_log(ctx, PJ_LOG_ERROR, + "Different unit for longitude and latitude offset"); + return nullptr; + } + if (!unitLatShift.empty()) { + if (unitLatShift == "arc-second") { + convFactorToRadian = ARC_SECOND_TO_RADIAN; + } else if (unitLatShift == "radian") { + convFactorToRadian = 1.0; + } else if (unitLatShift == "degree") { + convFactorToRadian = M_PI / 180.0; + } else { + pj_log(ctx, PJ_LOG_ERROR, "Unsupported unit %s", + unitLatShift.c_str()); + return nullptr; + } + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_name"); + + auto hgrid = internal::make_unique( + std::move(grid), idxLatShift, idxLonShift, convFactorToRadian, + positiveEast); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} + +// --------------------------------------------------------------------------- + std::unique_ptr HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (filename == "null") { @@ -849,6 +2073,12 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { strncmp(header + 0, "NUM_OREC", 8) == 0 && strncmp(header + 48, "GS_TYPE", 7) == 0) { return NTv2GridSet::open(ctx, fp, filename); + } else if (IsTIFF(header_size, + reinterpret_cast(header))) { + auto set = GTiffHGridShiftSet::open(ctx, fp, filename); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); diff --git a/src/grids.hpp b/src/grids.hpp index 65bf502b..e40528c3 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -45,6 +45,8 @@ struct ExtentAndRes { double resLat; // in radian bool fullWorldLongitude() const; + bool contains(const ExtentAndRes &other) const; + bool intersects(const ExtentAndRes &other) const; }; // --------------------------------------------------------------------------- @@ -87,6 +89,7 @@ class VerticalShiftGrid : public Grid { // --------------------------------------------------------------------------- class VerticalShiftGridSet { + protected: std::string m_name{}; std::string m_format{}; std::vector> m_grids{}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ae721d46..a7aac755 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -10,6 +10,7 @@ proj_add_gie_test("GDA" "gie/GDA.gie") proj_add_gie_test("4D-API-cs2cs-style" "gie/4D-API_cs2cs-style.gie") proj_add_gie_test("DHDN_ETRS89" "gie/DHDN_ETRS89.gie") proj_add_gie_test("Unitconvert" "gie/unitconvert.gie") +proj_add_gie_test("geotiff_grids" "gie/geotiff_grids.gie") # GIGS tests. Uncommented tests are expected to fail due to issues with # various projections. Should be investigated further and fixed. diff --git a/test/cli/test27 b/test/cli/test27 index bfc1cb0a..5825ed80 100755 --- a/test/cli/test27 +++ b/test/cli/test27 @@ -34,7 +34,7 @@ echo "Running ${0} using ${EXE}:" echo "============================================" OUT=proj_out27 -INIT_FILE=${PROJ_LIB}/nad27 +INIT_FILE=nad27 # echo "doing tests into file ${OUT}, please wait" # diff --git a/test/cli/test83 b/test/cli/test83 index cfb1365e..8c1293d0 100755 --- a/test/cli/test83 +++ b/test/cli/test83 @@ -35,7 +35,7 @@ echo "Running ${0} using ${EXE}:" echo "============================================" OUT=proj_out83 -INIT_FILE=${PROJ_LIB}/nad83 +INIT_FILE=nad83 # echo "doing tests into file ${OUT}, please wait" # diff --git a/test/cli/testdatumfile b/test/cli/testdatumfile index 29a40876..9bd12ce4 100755 --- a/test/cli/testdatumfile +++ b/test/cli/testdatumfile @@ -27,7 +27,11 @@ echo "Running ${0} using ${EXE}:" echo "============================================" mkdir "dir with \" space" -cp ${PROJ_LIB}/conus "dir with \" space/myconus" +if test -f "${PROJ_LIB}/conus"; then + cp "${PROJ_LIB}/conus" "dir with \" space/myconus" +else + cp "`dirname $0`/../../data/conus" "dir with \" space/myconus" +fi OUT=td_out #EXE=../src/cs2cs diff --git a/test/gie/4D-API_cs2cs-style.gie b/test/gie/4D-API_cs2cs-style.gie index 8d541823..3e4b9d2c 100644 --- a/test/gie/4D-API_cs2cs-style.gie +++ b/test/gie/4D-API_cs2cs-style.gie @@ -442,7 +442,6 @@ Test bugfix of https://github.com/OSGeo/proj.4/issues/1002 ------------------------------------------------------------------------------- operation +proj=latlong +ellps=WGS84 +geoidgrids=tests/test_nodata.gtx ------------------------------------------------------------------------------- -ignore pjd_err_failed_to_load_grid accept 4.05 52.1 0 expect 4.05 52.1 -10 ------------------------------------------------------------------------------- @@ -452,7 +451,6 @@ Test null grid with vgridshift ------------------------------------------------------------------------------- operation proj=vgridshift grids=tests/test_nodata.gtx,null ellps=GRS80 ------------------------------------------------------------------------------- -ignore pjd_err_failed_to_load_grid accept 4.05 52.1 0 expect 4.05 52.1 -10 diff --git a/test/gie/Makefile.am b/test/gie/Makefile.am index 44facd87..ff333a14 100644 --- a/test/gie/Makefile.am +++ b/test/gie/Makefile.am @@ -9,7 +9,8 @@ EXTRA_DIST = 4D-API_cs2cs-style.gie \ ellipsoid.gie \ more_builtins.gie \ unitconvert.gie \ - DHDN_ETRS89.gie + DHDN_ETRS89.gie \ + geotiff_grids.gie PROJ_LIB ?= ../../data @@ -40,4 +41,7 @@ unitconvert: unitconvert.gie DHDN_ETRS89: DHDN_ETRS89.gie PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< -check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 +geotiff_grids: geotiff_grids.gie + PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + +check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 geotiff_grids diff --git a/test/gie/geotiff_grids.gie b/test/gie/geotiff_grids.gie new file mode 100644 index 00000000..132ca6d4 --- /dev/null +++ b/test/gie/geotiff_grids.gie @@ -0,0 +1,283 @@ + +------------------------------------------------------------------------------- +=============================================================================== +Test GeoTIFF grids +=============================================================================== + + + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_pixelispoint.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_pixelisarea.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_deflate.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_deflate_floatingpointpredictor.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint16.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint16_with_scale_offset.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_int16.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_int32.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint32.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_float64.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +# The overview should be ignored +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_with_overview.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_in_second_channel.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigtiff.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigendian.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigendian_bigtiff.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bottomup_with_scale.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bottomup_with_matrix.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_with_subgrid.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 + +# In subgrid +accept 5.5 53.5 0 +expect 5.5 53.5 110.0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_nodata.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.05 52.1 0 +expect 4.05 52.1 10 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_invalid_channel_type.tif +multiplier=1 +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_unsupported_byte.tif +multiplier=1 +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + + + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_separate.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_strip.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_tiled.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_tiled_separate.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_positive_west.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_lon_shift_first.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_radian.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_degree.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +# The overview should be ignored +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_overview.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_extra_ifd_with_other_info.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +# Subset of NTv2_0.gsb +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_subgrid.tif +------------------------------------------------------------------------------- +# In subgrid ALbanff, of parent CAwest +accept -115.5416667 51.1666667 0 +expect -115.5427092888 51.1666899972 0 + +# In subgrid ONtronto, of parent CAeast +accept -80.5041667 44.5458333 0 +expect -80.50401615833 44.5458827236 0 +------------------------------------------------------------------------------- + +# Subset of NTv2_0.gsb +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_subgrid_no_grid_name.tif +------------------------------------------------------------------------------- +# In subgrid ALbanff, of parent CAwest +accept -115.5416667 51.1666667 0 +expect -115.5427092888 51.1666899972 0 + +# In subgrid ONtronto, of parent CAeast +accept -80.5041667 44.5458333 0 +expect -80.50401615833 44.5458827236 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_vgrid.tif +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + + + -- cgit v1.2.3 From 2804ee5ec0982a91d7fdb1304bd1490ebb9643a9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 10 Dec 2019 19:42:38 +0100 Subject: GTiff grid: use a block cache per dataset for faster access --- src/grids.cpp | 164 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 120 insertions(+), 44 deletions(-) diff --git a/src/grids.cpp b/src/grids.cpp index 010ddb22..471ef590 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -29,9 +29,11 @@ #ifndef FROM_PROJ_CPP #define FROM_PROJ_CPP #endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include "grids.hpp" #include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" #include "proj_internal.h" #include "tiffio.h" @@ -294,9 +296,67 @@ constexpr uint16 TIFFTAG_GDAL_NODATA = 42113; // --------------------------------------------------------------------------- +class BlockCache { + public: + void insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector &data); + std::shared_ptr> get(uint32 ifdIdx, + uint32 blockNumber); + + private: + struct Key { + uint32 ifdIdx; + uint32 blockNumber; + + Key(uint32 ifdIdxIn, uint32 blockNumberIn) + : ifdIdx(ifdIdxIn), blockNumber(blockNumberIn) {} + bool operator==(const Key &other) const { + return ifdIdx == other.ifdIdx && blockNumber == other.blockNumber; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return k.ifdIdx ^ (k.blockNumber << 16) ^ (k.blockNumber >> 16); + } + }; + + static constexpr int NUM_BLOCKS_AT_CROSSING_TILES = 4; + static constexpr int MAX_SAMPLE_COUNT = 3; + lru11::Cache< + Key, std::shared_ptr>, lru11::NullLock, + std::unordered_map< + Key, + typename std::list>>>::iterator, + KeyHasher>> + cache_{NUM_BLOCKS_AT_CROSSING_TILES * MAX_SAMPLE_COUNT}; +}; + +// --------------------------------------------------------------------------- + +void BlockCache::insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector &data) { + cache_.insert(Key(ifdIdx, blockNumber), + std::make_shared>(data)); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr> +BlockCache::get(uint32 ifdIdx, uint32 blockNumber) { + std::shared_ptr> ret; + cache_.tryGet(Key(ifdIdx, blockNumber), ret); + return ret; +} + +// --------------------------------------------------------------------------- + class GTiffGrid : public Grid { - PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset - TIFF *m_hTIFF; // owned by the belonging GTiffDataset + PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset + TIFF *m_hTIFF; // owned by the belonging GTiffDataset + BlockCache &m_cache; // owned by the belonging GTiffDataset + uint32 m_ifdIdx; TIFFDataType m_dt; uint16 m_samplesPerPixel; uint16 m_planarConfig; @@ -305,9 +365,6 @@ class GTiffGrid : public Grid { bool m_tiled; uint32 m_blockWidth = 0; uint32 m_blockHeight = 0; - mutable int m_lastSample = -1; - mutable int m_lastBlockX = -1; - mutable int m_lastBlockY = -1; mutable std::vector m_buffer{}; unsigned m_blocksPerRow = 0; unsigned m_blocksPerCol = 0; @@ -324,12 +381,14 @@ class GTiffGrid : public Grid { void getScaleOffset(double &scale, double &offset, uint16 sample) const; template - float readValue(uint32 offsetInBlock, uint16 sample) const; + float readValue(const std::vector &buffer, + uint32 offsetInBlock, uint16 sample) const; public: - GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, - const ExtentAndRes &extentIn, TIFFDataType dtIn, - uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx, + int widthIn, int heightIn, const ExtentAndRes &extentIn, + TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, + bool bottomUpIn); ~GTiffGrid() override; @@ -346,11 +405,13 @@ class GTiffGrid : public Grid { // --------------------------------------------------------------------------- -GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, int widthIn, int heightIn, +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, + uint32 ifdIdx, int widthIn, int heightIn, const ExtentAndRes &extentIn, TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn) - : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), m_dt(dtIn), + : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), + m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn), m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), m_tiled(TIFFIsTiled(hTIFF) != 0) { @@ -473,9 +534,10 @@ void GTiffGrid::getScaleOffset(double &scale, double &offset, // --------------------------------------------------------------------------- template -float GTiffGrid::readValue(uint32 offsetInBlock, uint16 sample) const { - const auto ptr = reinterpret_cast(m_buffer.data()); - assert(offsetInBlock < m_buffer.size() / sizeof(T)); +float GTiffGrid::readValue(const std::vector &buffer, + uint32 offsetInBlock, uint16 sample) const { + const auto ptr = reinterpret_cast(buffer.data()); + assert(offsetInBlock < buffer.size() / sizeof(T)); const auto val = ptr[offsetInBlock]; if (!m_hasNodata || static_cast(val) != m_noData) { double scale = 1; @@ -493,20 +555,6 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, float &out) const { assert(x >= 0 && yFromBottom >= 0 && x < m_width && yFromBottom < m_height); assert(sample < m_samplesPerPixel); - if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && - !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { - return false; - } - if (m_buffer.empty()) { - const auto blockSize = static_cast( - m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); - try { - m_buffer.resize(blockSize); - } catch (const std::exception &e) { - pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); - return false; - } - } const int blockX = x / m_blockWidth; @@ -518,12 +566,32 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, const int yTIFF = m_bottomUp ? yFromBottom : m_height - 1 - yFromBottom; const int blockY = yTIFF / m_blockHeight; - if (m_lastSample != static_cast(sample) || m_lastBlockX != blockX || - m_lastBlockY != blockY) { - uint32 blockId = blockY * m_blocksPerRow + blockX; - if (m_planarConfig == PLANARCONFIG_SEPARATE) { - blockId += sample * m_blocksPerCol * m_blocksPerRow; + uint32 blockId = blockY * m_blocksPerRow + blockX; + if (m_planarConfig == PLANARCONFIG_SEPARATE) { + blockId += sample * m_blocksPerCol * m_blocksPerRow; + } + + auto cachedBuffer = m_cache.get(m_ifdIdx, blockId); + std::vector *pBuffer = &m_buffer; + if (cachedBuffer != nullptr) { + // Safe as we don't access the cache before pBuffer is used + pBuffer = cachedBuffer.get(); + } else { + if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && + !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { + return false; } + if (m_buffer.empty()) { + const auto blockSize = static_cast( + m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); + try { + m_buffer.resize(blockSize); + } catch (const std::exception &e) { + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + return false; + } + } + if (m_tiled) { if (TIFFReadEncodedTile(m_hTIFF, blockId, m_buffer.data(), m_buffer.size()) == -1) { @@ -535,10 +603,15 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, return false; } } - m_lastSample = sample; - m_lastBlockX = blockX; - m_lastBlockY = blockY; + + try { + m_cache.insert(m_ifdIdx, blockId, m_buffer); + } catch (const std::exception &e) { + // Should normally not happen + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + } } + uint32 offsetInBlock = (x % m_blockWidth) + (yTIFF % m_blockHeight) * m_blockWidth; if (m_planarConfig == PLANARCONFIG_CONTIG) @@ -546,27 +619,27 @@ bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, switch (m_dt) { case TIFFDataType::Int16: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::UInt16: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::Int32: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::UInt32: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::Float32: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; case TIFFDataType::Float64: - out = readValue(offsetInBlock, sample); + out = readValue(*pBuffer, offsetInBlock, sample); break; } @@ -596,8 +669,10 @@ class GTiffDataset { PAFile m_fp; TIFF *m_hTIFF = nullptr; bool m_hasNextGrid = false; + uint32 m_ifdIdx = 0; toff_t m_nextDirOffset = 0; std::string m_filename{}; + BlockCache m_cache{}; GTiffDataset(const GTiffDataset &) = delete; GTiffDataset &operator=(const GTiffDataset &) = delete; @@ -948,8 +1023,9 @@ std::unique_ptr GTiffDataset::nextGrid() { } auto ret = std::unique_ptr( - new GTiffGrid(m_ctx, m_hTIFF, width, height, extent, dt, - samplesPerPixel, planarConfig, vRes < 0)); + new GTiffGrid(m_ctx, m_hTIFF, m_cache, m_ifdIdx, width, height, extent, + dt, samplesPerPixel, planarConfig, vRes < 0)); + m_ifdIdx++; m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); return ret; -- cgit v1.2.3 From e872984b46145fd03016fc53c6dac5843ba344f8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Dec 2019 16:41:06 +0100 Subject: grids.hpp: add a GenericShiftGridSet and GenericShiftGrid classes, that are going to be used by later commit --- src/grids.cpp | 299 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/grids.hpp | 53 +++++++++++ 2 files changed, 352 insertions(+) diff --git a/src/grids.cpp b/src/grids.cpp index 471ef590..eca0033a 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -2201,4 +2201,303 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, return nullptr; } +// --------------------------------------------------------------------------- + +class GTiffGenericGridShiftSet : public GenericShiftGridSet, + public GTiffDataset { + + GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) + : GTiffDataset(ctx, fp) {} + + public: + ~GTiffGenericGridShiftSet() override; + + static std::unique_ptr + open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); +}; + +// --------------------------------------------------------------------------- + +class GTiffGenericGrid : public GenericShiftGrid { + friend void insertIntoHierarchy( + PJ_CONTEXT *ctx, std::unique_ptr &&grid, + const std::string &gridName, const std::string &parentName, + std::vector> &topGrids, + std::map &mapGrids); + + std::unique_ptr m_grid; + + public: + GTiffGenericGrid(std::unique_ptr &&grid); + + ~GTiffGenericGrid() override; + + bool valueAt(int x, int y, int sample, float &out) const override; + + int samplesPerPixel() const override { return m_grid->samplesPerPixel(); } + + std::string unit(int sample) const override { + return m_grid->metadataItem("UNITTYPE", sample); + } + + std::string description(int sample) const override { + return m_grid->metadataItem("DESCRIPTION", sample); + } + + std::string metadataItem(const std::string &key, + int sample = -1) const override { + return m_grid->metadataItem(key, sample); + } + + void insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid); +}; + +// --------------------------------------------------------------------------- + +GTiffGenericGridShiftSet::~GTiffGenericGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::GTiffGenericGrid(std::unique_ptr &&grid) + : GenericShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + m_grid(std::move(grid)) {} + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::~GTiffGenericGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffGenericGrid::valueAt(int x, int y, int sample, float &out) const { + if (sample < 0 || + static_cast(sample) >= m_grid->samplesPerPixel()) + return false; + return m_grid->valueAt(static_cast(sample), x, y, out); +} + +// --------------------------------------------------------------------------- + +void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +class NullGenericShiftGrid : public GenericShiftGrid { + + public: + NullGenericShiftGrid() : GenericShiftGrid(3, 3, globalExtent()) {} + + bool isNullGrid() const override { return true; } + bool valueAt(int, int, int, float &out) const override; + + int samplesPerPixel() const override { return 0; } + + std::string unit(int) const override { return std::string(); } + + std::string description(int) const override { return std::string(); } + + std::string metadataItem(const std::string &, int) const override { + return std::string(); + } +}; + +// --------------------------------------------------------------------------- + +bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { + out = 0.0f; + return true; +} + +// --------------------------------------------------------------------------- + +std::unique_ptr +GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &filename) { + auto set = std::unique_ptr( + new GTiffGenericGridShiftSet(ctx, fp)); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + + std::map mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_name"); + + auto hgrid = internal::make_unique(std::move(grid)); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} + +// --------------------------------------------------------------------------- + +GenericShiftGrid::GenericShiftGrid(int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : Grid(widthIn, heightIn, extentIn) {} + +// --------------------------------------------------------------------------- + +GenericShiftGrid::~GenericShiftGrid() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::~GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +std::unique_ptr +GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { + if (filename == "null") { + auto set = + std::unique_ptr(new GenericShiftGridSet()); + set->m_name = filename; + set->m_format = "null"; + set->m_grids.push_back( + std::unique_ptr(new NullGenericShiftGrid())); + return set; + } + + PAFile fp; + if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + ctx->last_errno = 0; /* don't treat as a persistent error */ + return nullptr; + } + + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size; + if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != + sizeof(header)) { + pj_ctx_fclose(ctx, fp); + return nullptr; + } + pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + + if (IsTIFF(header_size, header)) { + auto set = GTiffGenericGridShiftSet::open(ctx, fp, filename); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; + } + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); + pj_ctx_fclose(ctx, fp); + return nullptr; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGrid::gridAt(double lon, double lat) const { + for (const auto &child : m_children) { + const auto &extentChild = child->extentAndRes(); + if ((extentChild.fullWorldLongitude() || + (lon >= extentChild.westLon && lon <= extentChild.eastLon)) && + lat >= extentChild.southLat && lat <= extentChild.northLat) { + return child->gridAt(lon, lat); + } + } + return this; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGridSet::gridAt(double lon, + double lat) const { + for (const auto &grid : m_grids) { + if (dynamic_cast(grid.get())) { + return grid.get(); + } + const auto &extent = grid->extentAndRes(); + if ((extent.fullWorldLongitude() || + (lon >= extent.westLon && lon <= extent.eastLon)) && + lat >= extent.southLat && lat <= extent.northLat) { + return grid->gridAt(lon, lat); + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { + std::string key("s"); + key += gridkey; + const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; + if (gridnames == nullptr) + return {}; + + auto listOfGridNames = internal::split(std::string(gridnames), ','); + ListOfGenericGrids grids; + for (const auto &gridnameStr : listOfGridNames) { + const char *gridname = gridnameStr.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = GenericShiftGridSet::open(P->ctx, gridname); + if (!gridSet) { + if (!canFail) { + pj_ctx_set_errno(P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return {}; + } + } else { + grids.emplace_back(std::move(gridSet)); + } + } + + return grids; +} + NS_PROJ_END diff --git a/src/grids.hpp b/src/grids.hpp index e40528c3..99683c39 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -154,11 +154,64 @@ class HorizontalShiftGridSet { // --------------------------------------------------------------------------- +class GenericShiftGrid : public Grid { + protected: + std::vector> m_children{}; + + public: + GenericShiftGrid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + + ~GenericShiftGrid() override; + + const GenericShiftGrid *gridAt(double lon, double lat) const; + + virtual std::string unit(int sample) const = 0; + + virtual std::string description(int sample) const = 0; + + virtual std::string metadataItem(const std::string &key, + int sample = -1) const = 0; + + virtual int samplesPerPixel() const = 0; + + // x = 0 is western-most column, y = 0 is southern-most line + virtual bool valueAt(int x, int y, int sample, float &out) const = 0; +}; + +// --------------------------------------------------------------------------- + +class GenericShiftGridSet { + protected: + std::string m_name{}; + std::string m_format{}; + std::vector> m_grids{}; + + GenericShiftGridSet(); + + public: + virtual ~GenericShiftGridSet(); + + static std::unique_ptr + open(PJ_CONTEXT *ctx, const std::string &filename); + + const std::string &name() const { return m_name; } + const std::string &format() const { return m_format; } + const std::vector> &grids() const { + return m_grids; + } + const GenericShiftGrid *gridAt(double lon, double lat) const; +}; + +// --------------------------------------------------------------------------- + typedef std::vector> ListOfHGrids; typedef std::vector> ListOfVGrids; +typedef std::vector> ListOfGenericGrids; ListOfVGrids proj_vgrid_init(PJ *P, const char *grids); ListOfHGrids proj_hgrid_init(PJ *P, const char *grids); +ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *grids); + double proj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, double vmultiplier); PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids &, PJ_LP lp); -- cgit v1.2.3 From 6d45bbc88bd58d7fc8a4ff7ec70bd2f23cb67aa1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Dec 2019 16:42:13 +0100 Subject: Add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid --- docs/source/operations/transformations/index.rst | 1 + .../operations/transformations/xyzgridshift.rst | 91 ++++++ docs/source/references.bib | 9 + src/Makefile.am | 1 + src/lib_proj.cmake | 1 + src/pj_list.h | 1 + src/transformations/xyzgridshift.cpp | 329 +++++++++++++++++++++ test/gie/geotiff_grids.gie | 24 ++ 8 files changed, 457 insertions(+) create mode 100644 docs/source/operations/transformations/xyzgridshift.rst create mode 100644 src/transformations/xyzgridshift.cpp diff --git a/docs/source/operations/transformations/index.rst b/docs/source/operations/transformations/index.rst index d4bbf4e0..6bd503d4 100644 --- a/docs/source/operations/transformations/index.rst +++ b/docs/source/operations/transformations/index.rst @@ -19,3 +19,4 @@ systems are based on different datums. molobadekas hgridshift vgridshift + xyzgridshift diff --git a/docs/source/operations/transformations/xyzgridshift.rst b/docs/source/operations/transformations/xyzgridshift.rst new file mode 100644 index 00000000..c49b7592 --- /dev/null +++ b/docs/source/operations/transformations/xyzgridshift.rst @@ -0,0 +1,91 @@ +.. _xyzgridshift: + +================================================================================ +Geocentric grid shift +================================================================================ + +.. versionadded:: 7.0.0 + +Geocentric translation using a grid shift + ++-----------------+-------------------------------------------------------------------+ +| **Domain** | 3D | ++-----------------+-------------------------------------------------------------------+ +| **Input type** | Cartesian coordinates | ++-----------------+-------------------------------------------------------------------+ +| **Output type** | Cartesian coordinates | ++-----------------+-------------------------------------------------------------------+ + +Perform a geocentric translation by bilinear interpolation of dx, dy, dz +translation values from a grid. The grid is referenced against either the +2D geographic CRS corresponding to the input (or sometimes output) CRS. + +This method is described (in French) in :cite:`NTF_88` +and as EPSG operation methode code 9655 in :cite:`IOGP2018` (§2.4.4.1.1 +France geocentric interpolation). + +The translation in the grids are added to the input coordinates in the forward direction, +and subtracted in the reverse direction. +By default (if grid_ref=input_crs), in the forward direction, the input coordinates +are converted to their geographic equivalent to directly read and interpolate from +the grid. In the reverse direction, an iterative method is used to be able to find +the grid locations to read. +If grid_ref=output_crs is used, then the reverse strategy is applied: iterative +method in the forward direction, and direct read in the reverse direction. + +Example +------------------------------------------------------------------------------- + +NTF to RGF93 transformation using gr3df97a.tif grid + +:: + + +proj=pipeline + +step +proj=push +v_3 + +step +proj=cart +ellps=clrk80ign + +step +proj=xyzgridshift +grids=gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 + +step +proj=cart +ellps=GRS80 +inv + +step +proj=pop +v_3 + +Parameters +################################################################################ + +The ellipsoid parameters should be the ones consistent with ``grid_ref``. +They are used to perform a geocentric to geographic conversion to find the +translation parameters. + + +Required ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. include:: ../options/ellps.rst + +.. option:: +grids= + + Comma-separated list of grids to load. If a grid is prefixed by an `@` the + grid is considered optional and PROJ will the not complain if the grid is + not available. + + Grids are expected to be in GeoTIFF format. If no metadata is provided, + the first, second and third samples are assumed to be the geocentric + translation along X, Y and Z axis respectively, in metres. + +Optional ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. option:: +grid_ref=input_crs/output_crs + + Specify in which CRS the grid is referenced to. The default value is + input_crs. That is the grid is referenced in the geographic CRS corresponding + to the input geocentric CRS. + + If output_crs is specified, the grid is referenced in the geographic CRS corresponding + to the output geocentric CRS. This is for example the case for the French + gr3df97a.tif grid converting from NTF to RGF93, but referenced against RGF93. + Thus in the forward direction (NTF->RGF93), an iterative method is used to find + the appropriate shift. + +.. option:: +multiplier= + + Specify the multiplier to apply to the grid values. Defaults to 1.0 + diff --git a/docs/source/references.bib b/docs/source/references.bib index 0668e10d..8e527ea1 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -230,6 +230,15 @@ Doi = {10.2312/PE/PG/PG2012short/005-010} } +@TechReport{NTF_88, + Title = {Grille de parametres de transformation de coordonnees - GR3DF97A - Notice d'utilisation}, + Author = {IGN}, + Institution = {Service de Geodesie et Nivellement, Institut Geographique National}, + Year = {1997}, + + Url = {https://geodesie.ign.fr/contenu/fichiers/documentation/algorithmes/notice/NTG_88.pdf} +} + @TechReport{ONeilLaubscher1976, Title = {Extended Studies of a Quadrilateralized Spherical Cube Earth Data Base}, Author = {E. M. O'Neill and R. E. Laubscher}, diff --git a/src/Makefile.am b/src/Makefile.am index f58e57fd..89f8dc0c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -183,6 +183,7 @@ libproj_la_SOURCES = \ transformations/horner.cpp \ transformations/molodensky.cpp \ transformations/vgridshift.cpp \ + transformations/xyzgridshift.cpp \ \ aasincos.cpp adjlon.cpp \ dmstor.cpp auth.cpp \ diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index f1547afe..f94f0562 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -198,6 +198,7 @@ set(SRC_LIBPROJ_TRANSFORMATIONS transformations/horner.cpp transformations/molodensky.cpp transformations/vgridshift.cpp + transformations/xyzgridshift.cpp ) set(SRC_LIBPROJ_ISO19111 diff --git a/src/pj_list.h b/src/pj_list.h index 0923bba8..9798a36b 100644 --- a/src/pj_list.h +++ b/src/pj_list.h @@ -170,3 +170,4 @@ PROJ_HEAD(weren, "Werenskiold I") PROJ_HEAD(wink1, "Winkel I") PROJ_HEAD(wink2, "Winkel II") PROJ_HEAD(wintri, "Winkel Tripel") +PROJ_HEAD(xyzgridshift, "XYZ grid shift") diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp new file mode 100644 index 00000000..f23c2588 --- /dev/null +++ b/src/transformations/xyzgridshift.cpp @@ -0,0 +1,329 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: Geocentric translation using a grid + * Author: Even Rouault, + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#define PJ_LIB__ + +#include +#include +#include +#include + +#include "proj_internal.h" +#include "grids.hpp" + +#include + +PROJ_HEAD(xyzgridshift, "Geocentric grid shift"); + +using namespace NS_PROJ; + +namespace { // anonymous namespace +struct xyzgridshiftData { + PJ *cart = nullptr; + bool grid_ref_is_input = true; + ListOfGenericGrids grids{}; + double multiplier = 1.0; +}; +} // anonymous namespace + + +// --------------------------------------------------------------------------- + +static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, + const PJ_LP& input) +{ + for( const auto& gridset: grids ) + { + auto grid = gridset->gridAt(input.lam, input.phi); + if( grid ) + return grid; + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static bool get_grid_values(PJ* P, + const xyzgridshiftData* Q, + const PJ_LP& lp, + double& dx, + double& dy, + double& dz) +{ + auto grid = findGrid(Q->grids, lp); + if( !grid ) { + return false; + } + if( grid->isNullGrid() ) { + dx = 0; + dy = 0; + dz = 0; + return true; + } + const auto samplesPerPixel = grid->samplesPerPixel(); + if( samplesPerPixel < 3 ) { + proj_log_error(P, "xyzgridshift: grid has not enough samples"); + return false; + } + int sampleX = 0; + int sampleY = 1; + int sampleZ = 2; + for( int i = 0; i < samplesPerPixel; i++ ) + { + const auto desc = grid->description(i); + if( desc == "x_translation") { + sampleX = i; + } else if( desc == "y_translation") { + sampleY = i; + } else if( desc == "z_translation") { + sampleZ = i; + } + } + const auto unit = grid->unit(sampleX); + if( !unit.empty() && unit != "metre" ) { + proj_log_error(P, "xyzgridshift: Only unit=metre currently handled"); + return false; + } + + const auto& extent = grid->extentAndRes(); + double grid_x = (lp.lam - extent.westLon) / extent.resLon; + double grid_y = (lp.phi - extent.southLat) / extent.resLat; + int ix = static_cast(grid_x); + int iy = static_cast(grid_y); + int ix2 = std::min(ix + 1, grid->width() - 1); + int iy2 = std::min(iy + 1, grid->height() - 1); + + float dx1, dy1, dz1; + if( !grid->valueAt(ix, iy, sampleX, dx1) || + !grid->valueAt(ix, iy, sampleY, dy1) || + !grid->valueAt(ix, iy, sampleZ, dz1) ) { + return false; + } + + float dx2, dy2, dz2; + if( !grid->valueAt(ix2, iy, sampleX, dx2) || + !grid->valueAt(ix2, iy, sampleY, dy2) || + !grid->valueAt(ix2, iy, sampleZ, dz2) ) { + return false; + } + + float dx3, dy3, dz3; + if( !grid->valueAt(ix, iy2, sampleX, dx3) || + !grid->valueAt(ix, iy2, sampleY, dy3) || + !grid->valueAt(ix, iy2, sampleZ, dz3) ) { + return false; + } + + float dx4, dy4, dz4; + if( !grid->valueAt(ix2, iy2, sampleX, dx4) || + !grid->valueAt(ix2, iy2, sampleY, dy4) || + !grid->valueAt(ix2, iy2, sampleZ, dz4) ) { + return false; + } + + double frct_lam = grid_x - ix; + double frct_phi = grid_y - iy; + double m10 = frct_lam; + double m11 = m10; + double m01 = 1. - frct_lam; + double m00 = m01; + m11 *= frct_phi; + m01 *= frct_phi; + frct_phi = 1. - frct_phi; + m00 *= frct_phi; + m10 *= frct_phi; + dx = (m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4) * Q->multiplier; + dy = (m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4) * Q->multiplier; + dz = (m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4) * Q->multiplier; + return true; +} + +// --------------------------------------------------------------------------- + +#define SQUARE(x) ((x)*(x)) + +// --------------------------------------------------------------------------- + +static PJ_COORD iterative_adjustment(PJ* P, + const xyzgridshiftData* Q, + const PJ_COORD& pointInit, + double factor) +{ + PJ_COORD point = pointInit; + for(int i = 0; i < 10; i++) { + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + + dx *= factor; + dy *= factor; + dz *= factor; + + const double err = SQUARE((point.xyz.x - pointInit.xyz.x) - dx) + + SQUARE((point.xyz.y - pointInit.xyz.y) - dy) + + SQUARE((point.xyz.z - pointInit.xyz.z) - dz); + + point.xyz.x = pointInit.xyz.x + dx; + point.xyz.y = pointInit.xyz.y + dy; + point.xyz.z = pointInit.xyz.z + dz; + if( err < 1e-10 ) { + break; + } + } + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_COORD direct_adjustment(PJ* P, + const xyzgridshiftData* Q, + PJ_COORD point, + double factor) +{ + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + point.xyz.x += factor * dx; + point.xyz.y += factor * dy; + point.xyz.z += factor * dz; + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { + auto Q = static_cast(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.lpz = lpz; + + if( Q->grid_ref_is_input ) { + point = direct_adjustment(P, Q, point, 1.0); + } + else { + point = iterative_adjustment(P, Q, point, 1.0); + } + + return point.xyz; +} + + +static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { + auto Q = static_cast(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.xyz = xyz; + + if( Q->grid_ref_is_input ) { + point = iterative_adjustment(P, Q, point, -1.0); + } + else { + point = direct_adjustment(P, Q, point, -1.0); + } + + return point.lpz; +} + +static PJ *destructor (PJ *P, int errlev) { + if (nullptr==P) + return nullptr; + + auto Q = static_cast(P->opaque); + if( Q ) + { + if (Q->cart) + Q->cart->destructor (Q->cart, errlev); + delete Q; + } + P->opaque = nullptr; + + return pj_default_destructor(P, errlev); +} + +PJ *TRANSFORMATION(xyzgridshift,0) { + auto Q = new xyzgridshiftData; + P->opaque = (void *) Q; + P->destructor = destructor; + + P->fwd4d = nullptr; + P->inv4d = nullptr; + P->fwd3d = forward_3d; + P->inv3d = reverse_3d; + P->fwd = nullptr; + P->inv = nullptr; + + P->left = PJ_IO_UNITS_CARTESIAN; + P->right = PJ_IO_UNITS_CARTESIAN; + + // Pass a dummy ellipsoid definition that will be overridden just afterwards + Q->cart = proj_create(P->ctx, "+proj=cart +a=1"); + if (Q->cart == nullptr) + return destructor(P, ENOMEM); + + /* inherit ellipsoid definition from P to Q->cart */ + pj_inherit_ellipsoid_def (P, Q->cart); + + const char* grid_ref = pj_param (P->ctx, P->params, "sgrid_ref").s; + if( grid_ref ) { + if (strcmp(grid_ref, "input_crs") == 0 ) { + // default + } else if (strcmp(grid_ref, "output_crs") == 0 ) { + // Convention use for example for NTF->RGF93 grid that contains + // delta x,y,z from NTF to RGF93, but the grid itself is referenced + // in RGF93 + Q->grid_ref_is_input = false; + } else { + proj_log_error(P, "xyzgridshift: unusupported value for grid_ref"); + return destructor (P, PJD_ERR_NO_ARGS); + } + } + + if (0==pj_param(P->ctx, P->params, "tgrids").i) { + proj_log_error(P, "xyzgridshift: +grids parameter missing."); + return destructor (P, PJD_ERR_NO_ARGS); + } + + /* multiplier for delta x,y,z */ + if (pj_param(P->ctx, P->params, "tmultiplier").i) { + Q->multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; + } + + Q->grids = proj_generic_grid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "xyzgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + + return P; +} diff --git a/test/gie/geotiff_grids.gie b/test/gie/geotiff_grids.gie index 132ca6d4..1fde45a2 100644 --- a/test/gie/geotiff_grids.gie +++ b/test/gie/geotiff_grids.gie @@ -280,4 +280,28 @@ expect failure errno failed_to_load_grid ------------------------------------------------------------------------------- +# IGNF:LAMBE to IGNF:LAMB93 using xyzgridshift operation +------------------------------------------------------------------------------- +operation +proj=pipeline + +step +inv +proj=lcc +lat_1=46.8 +lat_0=46.8 +lon_0=0 + +k_0=0.99987742 +x_0=600000 +y_0=2200000 +ellps=clrk80ign +pm=paris + +step +proj=push +v_3 + +step +proj=cart +ellps=clrk80ign + +step +proj=xyzgridshift +grids=gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 + +step +proj=cart +ellps=GRS80 +inv + +step +proj=pop +v_3 + +step +proj=lcc +lat_0=46.5 +lon_0=3 +lat_1=49 +lat_2=44 + +x_0=700000 +y_0=6600000 +ellps=GRS80 +------------------------------------------------------------------------------- +ignore pjd_err_failed_to_load_grid +tolerance 1 mm + +accept 814149.529 1887019.768 0 +expect 860690.804 6319036.849 0 +# If using ntf_r93.gsb, one gets: 860690.805 6319036.850 + +roundtrip 1 +------------------------------------------------------------------------------- + + -- cgit v1.2.3 From 3fa5d486b9fa2152b54566e8390df6719a76a0f4 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Dec 2019 18:20:02 +0100 Subject: .cirrus.yml: add libtiff dependency --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index f1ff7a2e..4f140e72 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,7 +16,7 @@ task: folder: $HOME/.ccache pkginstall_script: - - pkg install -y autoconf automake libtool pkgconf sqlite3 wget unzip ccache gmake + - pkg install -y autoconf automake libtool pkgconf sqlite3 tiff wget unzip ccache gmake download_grid_script: - wget https://download.osgeo.org/proj/proj-datumgrid-1.8.zip - (cd data && unzip -o ../proj-datumgrid-1.8.zip) -- cgit v1.2.3 From 8063c0b992af82154b3d9b637c78aa769a482f8a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 11 Dec 2019 20:34:15 +0100 Subject: test/gie/geotiff_grids.gie: use of subset of gr3df97a.tif for systematic testing --- data/Makefile.am | 1 + data/tests/subset_of_gr3df97a.tif | Bin 0 -> 3215 bytes test/gie/geotiff_grids.gie | 3 +-- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 data/tests/subset_of_gr3df97a.tif diff --git a/data/Makefile.am b/data/Makefile.am index 9a7c0c28..2a8eb285 100644 --- a/data/Makefile.am +++ b/data/Makefile.am @@ -79,6 +79,7 @@ EXTRA_DIST = GL27 nad.lst nad27 nad83 \ tests/test_hgrid_extra_ifd_with_other_info.tif \ tests/test_hgrid_with_subgrid.tif \ tests/test_hgrid_with_subgrid_no_grid_name.tif \ + tests/subset_of_gr3df97a.tif \ null \ generate_all_sql_in.cmake sql_filelist.cmake \ $(SQL_ORDERED_LIST) diff --git a/data/tests/subset_of_gr3df97a.tif b/data/tests/subset_of_gr3df97a.tif new file mode 100644 index 00000000..a98783f3 Binary files /dev/null and b/data/tests/subset_of_gr3df97a.tif differ diff --git a/test/gie/geotiff_grids.gie b/test/gie/geotiff_grids.gie index 1fde45a2..920fcf28 100644 --- a/test/gie/geotiff_grids.gie +++ b/test/gie/geotiff_grids.gie @@ -287,13 +287,12 @@ operation +proj=pipeline +k_0=0.99987742 +x_0=600000 +y_0=2200000 +ellps=clrk80ign +pm=paris +step +proj=push +v_3 +step +proj=cart +ellps=clrk80ign - +step +proj=xyzgridshift +grids=gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 + +step +proj=xyzgridshift +grids=tests/subset_of_gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 +step +proj=cart +ellps=GRS80 +inv +step +proj=pop +v_3 +step +proj=lcc +lat_0=46.5 +lon_0=3 +lat_1=49 +lat_2=44 +x_0=700000 +y_0=6600000 +ellps=GRS80 ------------------------------------------------------------------------------- -ignore pjd_err_failed_to_load_grid tolerance 1 mm accept 814149.529 1887019.768 0 -- cgit v1.2.3 From c165952bcf0135bc26b6cfea3ae387c51ba59fb0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 12 Dec 2019 19:27:57 +0100 Subject: grids.cpp: use 'parent_grid_name' as stated in RFC4 --- data/tests/test_hgrid_with_subgrid.tif | Bin 6943 -> 6943 bytes src/grids.cpp | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/tests/test_hgrid_with_subgrid.tif b/data/tests/test_hgrid_with_subgrid.tif index 2acc4581..46a8f2f4 100644 Binary files a/data/tests/test_hgrid_with_subgrid.tif and b/data/tests/test_hgrid_with_subgrid.tif differ diff --git a/src/grids.cpp b/src/grids.cpp index eca0033a..ca3ddfcc 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -1054,7 +1054,7 @@ insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr &&grid, std::map &mapGrids) { const auto &extent = grid->extentAndRes(); - // If we have one or both of grid_name and parent_name, try to use + // If we have one or both of grid_name and parent_grid_name, try to use // the names to recreate the hiearchy if (!gridName.empty()) { if (mapGrids.find(gridName) != mapGrids.end()) { @@ -1250,7 +1250,7 @@ GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } const std::string gridName = grid->metadataItem("grid_name"); - const std::string parentName = grid->metadataItem("parent_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); auto vgrid = internal::make_unique(std::move(grid), idxSample); @@ -2068,7 +2068,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } const std::string gridName = grid->metadataItem("grid_name"); - const std::string parentName = grid->metadataItem("parent_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); auto hgrid = internal::make_unique( std::move(grid), idxLatShift, idxLonShift, convFactorToRadian, @@ -2364,7 +2364,7 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } const std::string gridName = grid->metadataItem("grid_name"); - const std::string parentName = grid->metadataItem("parent_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); auto hgrid = internal::make_unique(std::move(grid)); -- cgit v1.2.3 From 49f8ea7ea4755144310722a927f1e6f788c39e51 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 13 Dec 2019 15:25:59 +0100 Subject: Add configure/CMake option, strongly discouraged, to disable TIFF support --- CMakeLists.txt | 11 ++++++++--- configure.ac | 20 +++++++++++++++----- src/Makefile.am | 2 +- src/grids.cpp | 34 ++++++++++++++++++++++++++++++++++ src/lib_proj.cmake | 9 +++++++-- 5 files changed, 65 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb8ae88f..97a0b90a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,9 +137,14 @@ ENDIF() # Check for libtiff ################################################################################ -find_package(TIFF REQUIRED) -if(NOT TIFF_FOUND) - message(SEND_ERROR "libtiff dependency not found!") +option(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED "Disable TIFF support (strongly discouraged !)" OFF) +mark_as_advanced(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) +if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) + find_package(TIFF REQUIRED) + if(NOT TIFF_FOUND) + message(SEND_ERROR "libtiff dependency not found!") + endif() + add_definitions(-DTIFF_ENABLED) endif() ################################################################################ diff --git a/configure.ac b/configure.ac index 7c04f60b..4aff6d74 100644 --- a/configure.ac +++ b/configure.ac @@ -300,15 +300,25 @@ dnl --------------------------------------------------------------------------- dnl Check for libtiff dnl --------------------------------------------------------------------------- -if test "x$TIFF_CFLAGS$TIFF_LIBS" = "x" ; then - if $PKG_CONFIG libtiff; then - PKG_CHECK_MODULES([TIFF], [libtiff]) - else - PKG_CHECK_MODULES([TIFF], [libtiff-4]) +AC_ARG_ENABLE([tiff-is-strongly-discouraged], + AS_HELP_STRING([--disable-tiff-is-strongly-discouraged], + [Disable TIFF support. Strongly discouraged !]), + [enable_tiff=no], + [enable_tiff=yes]) + +if test "x$enable_tiff" = "xyes" -o "x$enable_tiff" = ""; then + if test "x$TIFF_CFLAGS$TIFF_LIBS" = "x" ; then + if $PKG_CONFIG libtiff; then + PKG_CHECK_MODULES([TIFF], [libtiff]) + else + PKG_CHECK_MODULES([TIFF], [libtiff-4]) + fi fi + TIFF_ENABLED_FLAGS=-DTIFF_ENABLED fi AC_SUBST(TIFF_CFLAGS,$TIFF_CFLAGS) AC_SUBST(TIFF_LIBS,$TIFF_LIBS) +AC_SUBST(TIFF_ENABLED_FLAGS,$TIFF_ENABLED_FLAGS) dnl --------------------------------------------------------------------------- dnl Check for external Google Test diff --git a/src/Makefile.am b/src/Makefile.am index 89f8dc0c..9a4b3cae 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ + -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ diff --git a/src/grids.cpp b/src/grids.cpp index ca3ddfcc..090f7320 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -36,7 +36,9 @@ #include "proj/internal/lru_cache.hpp" #include "proj_internal.h" +#ifdef TIFF_ENABLED #include "tiffio.h" +#endif #include #include @@ -279,6 +281,8 @@ static bool IsTIFF(size_t header_size, const unsigned char *header) { (header[3] == 0x2B && header[2] == 0))); } +#ifdef TIFF_ENABLED + // --------------------------------------------------------------------------- enum class TIFFDataType { Int16, UInt16, Int32, UInt32, Float32, Float64 }; @@ -1044,6 +1048,8 @@ class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); }; +#endif // TIFF_ENABLED + // --------------------------------------------------------------------------- template @@ -1107,6 +1113,7 @@ insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr &&grid, } } +#ifdef TIFF_ENABLED // --------------------------------------------------------------------------- class GTiffVGrid : public VerticalShiftGrid { @@ -1260,6 +1267,7 @@ GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } return set; } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -1307,10 +1315,16 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_fseek(ctx, fp, SEEK_SET, 0); if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED auto set = GTiffVGridShiftSet::open(ctx, fp, filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; +#else + pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_ctx_fclose(ctx, fp); + return nullptr; +#endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); @@ -1816,6 +1830,8 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, return set; } +#ifdef TIFF_ENABLED + // --------------------------------------------------------------------------- class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { @@ -2079,6 +2095,7 @@ GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } return set; } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2151,10 +2168,16 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return NTv2GridSet::open(ctx, fp, filename); } else if (IsTIFF(header_size, reinterpret_cast(header))) { +#ifdef TIFF_ENABLED auto set = GTiffHGridShiftSet::open(ctx, fp, filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; +#else + pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_ctx_fclose(ctx, fp); + return nullptr; +#endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); @@ -2201,6 +2224,7 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, return nullptr; } +#ifdef TIFF_ENABLED // --------------------------------------------------------------------------- class GTiffGenericGridShiftSet : public GenericShiftGridSet, @@ -2298,6 +2322,7 @@ void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, m_children.emplace_back(std::move(subgrid)); } } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2329,6 +2354,8 @@ bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { // --------------------------------------------------------------------------- +#ifdef TIFF_ENABLED + std::unique_ptr GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename) { @@ -2373,6 +2400,7 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, } return set; } +#endif // TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2425,10 +2453,16 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_fseek(ctx, fp, SEEK_SET, 0); if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED auto set = GTiffGenericGridShiftSet::open(ctx, fp, filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; +#else + pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_ctx_fclose(ctx, fp); + return nullptr; +#endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index f94f0562..d4c95092 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -436,8 +436,13 @@ if(USE_THREAD AND Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) target_link_libraries(${PROJ_CORE_TARGET} ${CMAKE_THREAD_LIBS_INIT}) endif() -include_directories(${SQLITE3_INCLUDE_DIR} ${TIFF_INCLUDE_DIR}) -target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY} ${TIFF_LIBRARY}) +include_directories(${SQLITE3_INCLUDE_DIR}) +target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY}) + +if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) + include_directories(${TIFF_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${TIFF_LIBRARY}) +endif() if(MSVC) target_compile_definitions(${PROJ_CORE_TARGET} -- cgit v1.2.3 From b934fd3885fa017b0c7a5ca59083a7f52fdaafc0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 14 Dec 2019 00:15:27 +0100 Subject: Grid class: add a name() method --- src/grids.cpp | 117 ++++++++++++++++++++++++++++++++++------------------------ src/grids.hpp | 13 +++++-- 2 files changed, 77 insertions(+), 53 deletions(-) diff --git a/src/grids.cpp b/src/grids.cpp index 090f7320..8fe6d745 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -95,8 +95,9 @@ bool ExtentAndRes::intersects(const ExtentAndRes &other) const { // --------------------------------------------------------------------------- -Grid::Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn) - : m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} +Grid::Grid(const std::string &name, int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : m_name(name), m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} // --------------------------------------------------------------------------- @@ -104,9 +105,9 @@ Grid::~Grid() = default; // --------------------------------------------------------------------------- -VerticalShiftGrid::VerticalShiftGrid(int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} +VerticalShiftGrid::VerticalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -132,7 +133,7 @@ static ExtentAndRes globalExtent() { class NullVerticalShiftGrid : public VerticalShiftGrid { public: - NullVerticalShiftGrid() : VerticalShiftGrid(3, 3, globalExtent()) {} + NullVerticalShiftGrid() : VerticalShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } bool valueAt(int, int, float &out) const override; @@ -156,17 +157,19 @@ class GTXVerticalShiftGrid : public VerticalShiftGrid { GTXVerticalShiftGrid &operator=(const GTXVerticalShiftGrid &) = delete; public: - GTXVerticalShiftGrid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, + GTXVerticalShiftGrid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, + int widthIn, int heightIn, const ExtentAndRes &extentIn) - : VerticalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), m_fp(fp) { - } + : VerticalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(fp) {} ~GTXVerticalShiftGrid() override; bool valueAt(int x, int y, float &out) const override; bool isNodata(float val, double multiplier) const override; - static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, PAFile fp); + static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &name); }; // --------------------------------------------------------------------------- @@ -175,7 +178,8 @@ GTXVerticalShiftGrid::~GTXVerticalShiftGrid() { pj_ctx_fclose(m_ctx, m_fp); } // --------------------------------------------------------------------------- -GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp) { +GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, + const std::string &name) { unsigned char header[40]; /* -------------------------------------------------------------------- */ @@ -231,7 +235,7 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp) { extent.eastLon = (xorigin + xstep * (columns - 1)) * DEG_TO_RAD; extent.northLat = (yorigin + ystep * (rows - 1)) * DEG_TO_RAD; - return new GTXVerticalShiftGrid(ctx, fp, columns, rows, extent); + return new GTXVerticalShiftGrid(ctx, fp, name, columns, rows, extent); } // --------------------------------------------------------------------------- @@ -390,9 +394,9 @@ class GTiffGrid : public Grid { public: GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx, - int widthIn, int heightIn, const ExtentAndRes &extentIn, - TIFFDataType dtIn, uint16 samplesPerPixelIn, uint16 planarConfig, - bool bottomUpIn); + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); ~GTiffGrid() override; @@ -410,11 +414,11 @@ class GTiffGrid : public Grid { // --------------------------------------------------------------------------- GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, - uint32 ifdIdx, int widthIn, int heightIn, - const ExtentAndRes &extentIn, TIFFDataType dtIn, - uint16 samplesPerPixelIn, uint16 planarConfig, - bool bottomUpIn) - : Grid(widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), + uint32 ifdIdx, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn, + TIFFDataType dtIn, uint16 samplesPerPixelIn, + uint16 planarConfig, bool bottomUpIn) + : Grid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn), m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), @@ -512,6 +516,11 @@ GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, } catch (const std::exception &) { } } + + auto oIter = m_metadata.find(std::pair(-1, "grid_name")); + if (oIter != m_metadata.end()) { + m_name += ", " + oIter->second; + } } // --------------------------------------------------------------------------- @@ -1026,9 +1035,9 @@ std::unique_ptr GTiffDataset::nextGrid() { return nullptr; } - auto ret = std::unique_ptr( - new GTiffGrid(m_ctx, m_hTIFF, m_cache, m_ifdIdx, width, height, extent, - dt, samplesPerPixel, planarConfig, vRes < 0)); + auto ret = std::unique_ptr(new GTiffGrid( + m_ctx, m_hTIFF, m_cache, m_ifdIdx, m_filename, width, height, extent, + dt, samplesPerPixel, planarConfig, vRes < 0)); m_ifdIdx++; m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); @@ -1149,7 +1158,8 @@ GTiffVGridShiftSet::~GTiffVGridShiftSet() = default; // --------------------------------------------------------------------------- GTiffVGrid::GTiffVGrid(std::unique_ptr &&grid, uint16 idxSample) - : VerticalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + : VerticalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), m_grid(std::move(grid)), m_idxSample(idxSample) {} // --------------------------------------------------------------------------- @@ -1289,7 +1299,7 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return nullptr; } if (ends_with(filename, "gtx") || ends_with(filename, "GTX")) { - auto grid = GTXVerticalShiftGrid::open(ctx, fp); + auto grid = GTXVerticalShiftGrid::open(ctx, fp, filename); if (!grid) { pj_ctx_fclose(ctx, fp); return nullptr; @@ -1321,7 +1331,8 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else - pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); pj_ctx_fclose(ctx, fp); return nullptr; #endif @@ -1366,9 +1377,10 @@ const VerticalShiftGrid *VerticalShiftGridSet::gridAt(double lon, // --------------------------------------------------------------------------- -HorizontalShiftGrid::HorizontalShiftGrid(int widthIn, int heightIn, +HorizontalShiftGrid::HorizontalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -1387,7 +1399,8 @@ HorizontalShiftGridSet::~HorizontalShiftGridSet() = default; class NullHorizontalShiftGrid : public HorizontalShiftGrid { public: - NullHorizontalShiftGrid() : HorizontalShiftGrid(3, 3, globalExtent()) {} + NullHorizontalShiftGrid() + : HorizontalShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } @@ -1421,9 +1434,9 @@ class NTv1Grid : public HorizontalShiftGrid { NTv1Grid &operator=(const NTv1Grid &) = delete; public: - NTv1Grid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), + NTv1Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_fp(fp) {} ~NTv1Grid() override; @@ -1496,7 +1509,7 @@ NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, const int rows = static_cast( fabs((extent.northLat - extent.southLat) / extent.resLat + 0.5) + 1); - return new NTv1Grid(ctx, fp, columns, rows, extent); + return new NTv1Grid(ctx, fp, filename, columns, rows, extent); } // --------------------------------------------------------------------------- @@ -1535,9 +1548,9 @@ class CTable2Grid : public HorizontalShiftGrid { CTable2Grid &operator=(const CTable2Grid &) = delete; public: - CTable2Grid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), + CTable2Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, + int widthIn, int heightIn, const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_fp(fp) {} ~CTable2Grid() override; @@ -1602,7 +1615,7 @@ CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, extent.eastLon = extent.westLon + (width - 1) * extent.resLon; extent.northLat = extent.southLat + (height - 1) * extent.resLon; - return new CTable2Grid(ctx, fp, width, height, extent); + return new CTable2Grid(ctx, fp, filename, width, height, extent); } // --------------------------------------------------------------------------- @@ -1665,8 +1678,9 @@ class NTv2Grid : public HorizontalShiftGrid { NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, PAFile fp, unsigned long long offsetIn, bool mustSwapIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_name(nameIn), - m_ctx(ctx), m_fp(fp), m_offset(offsetIn), m_mustSwap(mustSwapIn) {} + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), + m_name(nameIn), m_ctx(ctx), m_fp(fp), m_offset(offsetIn), + m_mustSwap(mustSwapIn) {} bool valueAt(int, int, float &lonShift, float &latShift) const override; }; @@ -1811,8 +1825,9 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, } auto offset = pj_ctx_ftell(ctx, fp); - auto grid = std::unique_ptr(new NTv2Grid( - gridName, ctx, fp, offset, must_swap, columns, rows, extent)); + auto grid = std::unique_ptr( + new NTv2Grid(filename + ", " + gridName, ctx, fp, offset, must_swap, + columns, rows, extent)); std::string parentName; parentName.assign(header + 24, 8); auto iter = mapGrids.find(parentName); @@ -1881,7 +1896,8 @@ GTiffHGridShiftSet::~GTiffHGridShiftSet() = default; GTiffHGrid::GTiffHGrid(std::unique_ptr &&grid, uint16 idxLatShift, uint16 idxLonShift, double convFactorToRadian, bool positiveEast) - : HorizontalShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + : HorizontalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), m_grid(std::move(grid)), m_idxLatShift(idxLatShift), m_idxLonShift(idxLonShift), m_convFactorToRadian(convFactorToRadian), m_positiveEast(positiveEast) {} @@ -2174,7 +2190,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else - pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); pj_ctx_fclose(ctx, fp); return nullptr; #endif @@ -2284,7 +2301,8 @@ GTiffGenericGridShiftSet::~GTiffGenericGridShiftSet() = default; // --------------------------------------------------------------------------- GTiffGenericGrid::GTiffGenericGrid(std::unique_ptr &&grid) - : GenericShiftGrid(grid->width(), grid->height(), grid->extentAndRes()), + : GenericShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), m_grid(std::move(grid)) {} // --------------------------------------------------------------------------- @@ -2329,7 +2347,7 @@ void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, class NullGenericShiftGrid : public GenericShiftGrid { public: - NullGenericShiftGrid() : GenericShiftGrid(3, 3, globalExtent()) {} + NullGenericShiftGrid() : GenericShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } bool valueAt(int, int, int, float &out) const override; @@ -2404,9 +2422,9 @@ GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, // --------------------------------------------------------------------------- -GenericShiftGrid::GenericShiftGrid(int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} +GenericShiftGrid::GenericShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -2459,7 +2477,8 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else - pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); pj_ctx_fclose(ctx, fp); return nullptr; #endif diff --git a/src/grids.hpp b/src/grids.hpp index 99683c39..0f595754 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -53,11 +53,13 @@ struct ExtentAndRes { class Grid { protected: + std::string m_name; int m_width; int m_height; ExtentAndRes m_extent; - Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + Grid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); public: virtual ~Grid(); @@ -65,6 +67,7 @@ class Grid { int width() const { return m_width; } int height() const { return m_height; } const ExtentAndRes &extentAndRes() const { return m_extent; } + const std::string &name() const { return m_name; } virtual bool isNullGrid() const { return false; } }; @@ -76,7 +79,8 @@ class VerticalShiftGrid : public Grid { std::vector> m_children{}; public: - VerticalShiftGrid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + VerticalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); const VerticalShiftGrid *gridAt(double lon, double lat) const; @@ -117,7 +121,7 @@ class HorizontalShiftGrid : public Grid { std::vector> m_children{}; public: - HorizontalShiftGrid(int widthIn, int heightIn, + HorizontalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn); ~HorizontalShiftGrid() override; @@ -159,7 +163,8 @@ class GenericShiftGrid : public Grid { std::vector> m_children{}; public: - GenericShiftGrid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + GenericShiftGrid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); ~GenericShiftGrid() override; -- cgit v1.2.3 From f36d2374268608d276da45c96a7b4792ab8ffdb5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 14 Dec 2019 00:16:14 +0100 Subject: Horizontal grid shift: fix issue on iterative inverse computation when switching between (sub)grids (fixes #1663) Given in.txt with 53.999759140 5.144478208 252.6995 Before the fix, cct -t 0 -d 4 +proj=pipeline +step +proj=axisswap +order=2,1,3,4 +step +proj=hgridshift +inv +grids=rdtrans2018.gsb +step +proj=vgridshift +grids=naptrans2018.gtx +step +proj=sterea +lat_0=52.156160556 +lon_0=5.387638889 +k=0.9999079 +x_0=155000 +y_0=463000 +ellps=bessel in.txt returned: 139079.8814 668306.0302 212.1724 0.0000 It now returns: 139079.8850 668306.0458 212.1724 0.0000 which meets with the 1mm accuracy the expected result of test point ``` 30010049 53.999759140 5.144478208 252.6995 139079.8850 668306.0460 212.1723 ``` --- src/apply_gridshift.cpp | 51 ++++++++++++++++++++++++++++++++----------------- test/cli/ntv2_out.dist | 3 +++ test/cli/testntv2 | 8 ++++++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp index c786a50a..b994474b 100644 --- a/src/apply_gridshift.cpp +++ b/src/apply_gridshift.cpp @@ -36,6 +36,8 @@ #include #include +#include + #include "proj.h" #include "proj_internal.h" #include "proj/internal/internal.hpp" @@ -191,7 +193,8 @@ static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { #define TOL 1e-12 static -PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { +PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* grid, + const ListOfHGrids& grids) { PJ_LP t, tb,del, dif; int i = MAX_ITERATIONS; const double toltol = TOL*TOL; @@ -201,9 +204,9 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { /* normalize input to ll origin */ tb = in; - const auto& extent = grid->extentAndRes(); - tb.lam -= extent.westLon; - tb.phi -= extent.southLat; + const auto* extent = &(grid->extentAndRes()); + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; tb.lam = adjlon (tb.lam - M_PI) + M_PI; @@ -223,17 +226,31 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { do { del = nad_intr(t, grid); - /* This case used to return failure, but I have - changed it to return the first order approximation - of the inverse shift. This avoids cases where the - grid shift *into* this grid came from another grid. - While we aren't returning optimally correct results - I feel a close result in this case is better than - no result. NFW - To demonstrate use -112.5839956 49.4914451 against - the NTv2 grid shift file from Canada. */ + /* We can possibly go outside of the initial guessed grid, so try */ + /* to fetch a new grid into which iterate... */ if (del.lam == HUGE_VAL) - break; + { + PJ_LP lp; + lp.lam = t.lam + extent->westLon; + lp.phi = t.phi + extent->southLat; + auto newGrid = findGrid(grids, lp); + if( newGrid == nullptr || newGrid == grid || newGrid->isNullGrid() ) + break; + pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", + grid->name().c_str(), + newGrid->name().c_str()); + grid = newGrid; + extent = &(grid->extentAndRes()); + t.lam = lp.lam - extent->westLon; + t.phi = lp.phi - extent->southLat; + tb = in; + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + tb.lam = adjlon (tb.lam - M_PI) + M_PI; + dif.lam = std::numeric_limits::max(); + dif.phi = std::numeric_limits::max(); + continue; + } dif.lam = t.lam + del.lam - tb.lam; dif.phi = t.phi + del.phi - tb.phi; @@ -254,8 +271,8 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { if (del.lam==HUGE_VAL && getenv ("PROJ_DEBUG")) fprintf (stderr, "Inverse grid shift iteration failed, presumably at grid edge.\nUsing first approximation.\n"); - in.lam = adjlon (t.lam + extent.westLon); - in.phi = t.phi + extent.southLat; + in.lam = adjlon (t.lam + extent->westLon); + in.phi = t.phi + extent->southLat; return in; } @@ -311,7 +328,7 @@ PJ_LP proj_hgrid_apply_internal(PJ_CONTEXT* ctx, } int inverse = direction == PJ_FWD ? 0 : 1; - out = nad_cvt(lp, inverse, grid); + out = nad_cvt(ctx, lp, inverse, grid, grids); if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); diff --git a/test/cli/ntv2_out.dist b/test/cli/ntv2_out.dist index 9a97f9cf..650a69d8 100644 --- a/test/cli/ntv2_out.dist +++ b/test/cli/ntv2_out.dist @@ -9,3 +9,6 @@ Try with NTv2 and NTv1 together ... falls back to NTv1 99d00'00.000"W 65d00'00.000"N 0.0 99d0'1.5885"W 65d0'1.3482"N 0.000 111d00'00.000"W 46d00'00.000"N 0.0 111d0'3.1897"W 45d59'59.7489"N 0.000 111d00'00.000"W 47d30'00.000"N 0.0 111d0'2.7989"W 47d29'59.9896"N 0.000 +############################################################## +Switching between NTv2 subgrids +-112.5839956 49.4914451 0 -112.58307487 49.49145197 0.00000000 diff --git a/test/cli/testntv2 b/test/cli/testntv2 index 73371dbe..2a31304e 100755 --- a/test/cli/testntv2 +++ b/test/cli/testntv2 @@ -52,6 +52,14 @@ $EXE +proj=latlong +ellps=clrk66 +nadgrids=ntv2_0.gsb,ntv1_can.dat,conus \ 111d00'00.000"W 46d00'00.000"N 0.0 111d00'00.000"W 47d30'00.000"N 0.0 EOF + +echo "##############################################################" >> ${OUT} +echo Switching between NTv2 subgrids >> ${OUT} +# Initial guess is in ALraymnd, going to parent CAwest afterwards +$EXE +proj=latlong +datum=NAD83 +to +proj=latlong +ellps=clrk66 +nadgrids=ntv2_0.gsb -E -d 8 >>${OUT} < Date: Wed, 18 Dec 2019 21:00:00 +0100 Subject: Rename PJ_CONTEXT::fileapi member as fileapi_legacy --- src/ctx.cpp | 8 ++++---- src/fileapi.cpp | 10 +++++----- src/proj_internal.h | 6 ++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/ctx.cpp b/src/ctx.cpp index bcb6e1cc..4536582d 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -95,7 +95,7 @@ projCtx_t projCtx_t::createDefault() projCtx_t ctx; ctx.debug_level = PJ_LOG_NONE; ctx.logger = pj_stderr_logger; - ctx.fileapi = pj_get_default_fileapi(); + ctx.fileapi_legacy = pj_get_default_fileapi(); if( getenv("PROJ_DEBUG") != nullptr ) { @@ -133,7 +133,7 @@ projCtx_t::projCtx_t(const projCtx_t& other) debug_level = other.debug_level; logger = other.logger; logger_app_data = other.logger_app_data; - fileapi = other.fileapi; + fileapi_legacy = other.fileapi_legacy; epsg_file_exists = other.epsg_file_exists; set_search_paths(other.search_paths); file_finder = other.file_finder; @@ -268,7 +268,7 @@ void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) { if (nullptr==ctx) return; - ctx->fileapi = fileapi; + ctx->fileapi_legacy = fileapi; } /************************************************************************/ @@ -280,6 +280,6 @@ projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) { if (nullptr==ctx) return nullptr; - return ctx->fileapi; + return ctx->fileapi_legacy; } diff --git a/src/fileapi.cpp b/src/fileapi.cpp index 70c7b5de..f39d94bb 100644 --- a/src/fileapi.cpp +++ b/src/fileapi.cpp @@ -141,7 +141,7 @@ static void stdio_fclose(PAFile file) PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) { - return ctx->fileapi->FOpen(ctx, filename, access); + return ctx->fileapi_legacy->FOpen(ctx, filename, access); } /************************************************************************/ @@ -149,7 +149,7 @@ PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) /************************************************************************/ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile file) { - return ctx->fileapi->FRead(buffer, size, nmemb, file); + return ctx->fileapi_legacy->FRead(buffer, size, nmemb, file); } /************************************************************************/ @@ -157,7 +157,7 @@ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile /************************************************************************/ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) { - return ctx->fileapi->FSeek(file, offset, whence); + return ctx->fileapi_legacy->FSeek(file, offset, whence); } /************************************************************************/ @@ -165,7 +165,7 @@ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) /************************************************************************/ long pj_ctx_ftell(projCtx ctx, PAFile file) { - return ctx->fileapi->FTell(file); + return ctx->fileapi_legacy->FTell(file); } /************************************************************************/ @@ -173,7 +173,7 @@ long pj_ctx_ftell(projCtx ctx, PAFile file) /************************************************************************/ void pj_ctx_fclose(projCtx ctx, PAFile file) { - ctx->fileapi->FClose(file); + ctx->fileapi_legacy->FClose(file); } /************************************************************************/ diff --git a/src/proj_internal.h b/src/proj_internal.h index 7d826414..93134946 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -214,9 +214,6 @@ size_t pj_trim_argc (char *args); char **pj_trim_argv (size_t argc, char *args); char *pj_make_args (size_t argc, char **argv); -/* Lowest level: Minimum support for fileapi */ -void proj_fileapi_set (PJ *P, void *fileapi); - typedef struct { double r, i; } COMPLEX; /* Forward declarations and typedefs for stuff needed inside the PJ object */ @@ -664,6 +661,7 @@ struct FACTORS { /* NOTE: Remember to update src/strerrno.cpp, src/apps/gie.cpp and transient_error in */ /* src/transform.cpp when adding new value */ +// Legacy struct projFileAPI_t; struct projCppContext; @@ -674,7 +672,7 @@ struct projCtx_t { int debug_level = 0; void (*logger)(void *, int, const char *) = nullptr; void *logger_app_data = nullptr; - struct projFileAPI_t *fileapi = nullptr; + struct projFileAPI_t *fileapi_legacy = nullptr; // for proj_api.h legacy API struct projCppContext* cpp_context = nullptr; /* internal context for C++ code */ int use_proj4_init_rules = -1; /* -1 = unknown, 0 = no, 1 = yes */ int epsg_file_exists = -1; /* -1 = unknown, 0 = no, 1 = yes */ -- cgit v1.2.3 From 7baf00c5bc96e1bb0ea2cb1a9495d2b5d703bdbc Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2019 23:34:00 +0100 Subject: Add test for legacy proj_api.h fileapi before further reworks --- test/unit/pj_transform_test.cpp | 91 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/test/unit/pj_transform_test.cpp b/test/unit/pj_transform_test.cpp index b3a061b4..5ca8dcce 100644 --- a/test/unit/pj_transform_test.cpp +++ b/test/unit/pj_transform_test.cpp @@ -614,4 +614,95 @@ TEST(proj_api_h, pj_set_finder) { pj_set_finder(nullptr); } +// --------------------------------------------------------------------------- + +TEST(proj_api_h, default_fileapi) { + auto ctx = pj_ctx_alloc(); + auto fid = pj_open_lib(ctx, "proj.db", "rb"); + ASSERT_NE(fid, nullptr); + char header[6]; + ASSERT_EQ(pj_ctx_fread(ctx, header, 1, 6, fid), 6U); + ASSERT_TRUE(memcmp(header, "SQLite", 6) == 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 6); + ASSERT_EQ(pj_ctx_fseek(ctx, fid, 0, SEEK_SET), 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 0); + pj_ctx_fclose(ctx, fid); + pj_ctx_free(ctx); +} + +// --------------------------------------------------------------------------- + +struct Spy { + bool gotInMyFOpen = false; + bool gotInMyFRead = false; + bool gotInMyFSeek = false; + bool gotInMyFTell = false; + bool gotInMyFClose = false; +}; + +struct MyFile { + FILE *fp; + Spy *spy; +}; + +static PAFile myFOpen(projCtx ctx, const char *filename, const char *access) { + FILE *fp = fopen(filename, access); + if (!fp) + return nullptr; + MyFile *myF = new MyFile; + myF->spy = (Spy *)pj_ctx_get_app_data(ctx); + myF->spy->gotInMyFOpen = true; + myF->fp = fp; + return reinterpret_cast(myF); +} + +static size_t myFRead(void *buffer, size_t size, size_t nmemb, PAFile file) { + MyFile *myF = reinterpret_cast(file); + myF->spy->gotInMyFRead = true; + return fread(buffer, size, nmemb, myF->fp); +} + +static int myFSeek(PAFile file, long offset, int whence) { + MyFile *myF = reinterpret_cast(file); + myF->spy->gotInMyFSeek = true; + return fseek(myF->fp, offset, whence); +} + +static long myFTell(PAFile file) { + MyFile *myF = reinterpret_cast(file); + myF->spy->gotInMyFTell = true; + return ftell(myF->fp); +} + +static void myFClose(PAFile file) { + MyFile *myF = reinterpret_cast(file); + myF->spy->gotInMyFClose = true; + fclose(myF->fp); + delete myF; +} + +TEST(proj_api_h, custom_fileapi) { + auto ctx = pj_ctx_alloc(); + Spy spy; + pj_ctx_set_app_data(ctx, &spy); + projFileAPI myAPI = {myFOpen, myFRead, myFSeek, myFTell, myFClose}; + pj_ctx_set_fileapi(ctx, &myAPI); + EXPECT_EQ(pj_ctx_get_fileapi(ctx), &myAPI); + auto fid = pj_open_lib(ctx, "proj.db", "rb"); + ASSERT_NE(fid, nullptr); + char header[6]; + ASSERT_EQ(pj_ctx_fread(ctx, header, 1, 6, fid), 6U); + ASSERT_TRUE(memcmp(header, "SQLite", 6) == 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 6); + ASSERT_EQ(pj_ctx_fseek(ctx, fid, 0, SEEK_SET), 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 0); + pj_ctx_fclose(ctx, fid); + pj_ctx_free(ctx); + EXPECT_TRUE(spy.gotInMyFOpen); + EXPECT_TRUE(spy.gotInMyFRead); + EXPECT_TRUE(spy.gotInMyFSeek); + EXPECT_TRUE(spy.gotInMyFTell); + EXPECT_TRUE(spy.gotInMyFClose); +} + } // namespace -- cgit v1.2.3 From 2dfe2cdd1f4e963e6faaccd5ca29bc6d8fe4ae30 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2019 23:38:16 +0100 Subject: Add a FileManager and File class --- scripts/reformat_cpp.sh | 3 +- src/Makefile.am | 4 +- src/ctx.cpp | 25 ------- src/fileapi.cpp | 26 ++++++++ src/filemanager.cpp | 173 ++++++++++++++++++++++++++++++++++++++++++++++++ src/filemanager.hpp | 68 +++++++++++++++++++ src/lib_proj.cmake | 2 + src/open_lib.cpp | 65 ++++++++++++++---- 8 files changed, 327 insertions(+), 39 deletions(-) create mode 100644 src/filemanager.cpp create mode 100644 src/filemanager.hpp diff --git a/scripts/reformat_cpp.sh b/scripts/reformat_cpp.sh index 899e665f..ca20e0d8 100755 --- a/scripts/reformat_cpp.sh +++ b/scripts/reformat_cpp.sh @@ -17,7 +17,8 @@ TOPDIR="$SCRIPT_DIR/.." for i in "$TOPDIR"/include/proj/*.hpp "$TOPDIR"/include/proj/internal/*.hpp \ "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp "$TOPDIR"/src/apps/projinfo.cpp \ - "$TOPDIR"/src/tracing.cpp "$TOPDIR"/src/grids.hpp "$TOPDIR"/src/grids.cpp; do + "$TOPDIR"/src/tracing.cpp "$TOPDIR"/src/grids.hpp "$TOPDIR"/src/grids.cpp \ + "$TOPDIR"/src/filemanager.hpp "$TOPDIR"/src/filemanager.cpp ; do if ! echo "$i" | grep -q "lru_cache.hpp"; then "$SCRIPT_DIR"/reformat.sh "$i"; fi diff --git a/src/Makefile.am b/src/Makefile.am index 9a4b3cae..f5cabe5e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,7 +215,9 @@ libproj_la_SOURCES = \ tracing.cpp \ \ grids.hpp \ - grids.cpp + grids.cpp \ + filemanager.hpp \ + filemanager.cpp # The sed hack is to please MSVC diff --git a/src/ctx.cpp b/src/ctx.cpp index 4536582d..da90d4d7 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -258,28 +258,3 @@ void *pj_ctx_get_app_data( projCtx ctx ) return nullptr; return ctx->logger_app_data; } - -/************************************************************************/ -/* pj_ctx_set_fileapi() */ -/************************************************************************/ - -void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) - -{ - if (nullptr==ctx) - return; - ctx->fileapi_legacy = fileapi; -} - -/************************************************************************/ -/* pj_ctx_get_fileapi() */ -/************************************************************************/ - -projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) - -{ - if (nullptr==ctx) - return nullptr; - return ctx->fileapi_legacy; -} - diff --git a/src/fileapi.cpp b/src/fileapi.cpp index f39d94bb..70be2502 100644 --- a/src/fileapi.cpp +++ b/src/fileapi.cpp @@ -34,6 +34,7 @@ #include "proj.h" #include "proj_internal.h" +#include "filemanager.hpp" static PAFile stdio_fopen(projCtx ctx, const char *filename, const char *access); @@ -212,3 +213,28 @@ char *pj_ctx_fgets(projCtx ctx, char *line, int size, PAFile file) } return line; } + +/************************************************************************/ +/* pj_ctx_set_fileapi() */ +/************************************************************************/ + +void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) + +{ + if (nullptr==ctx) + return; + ctx->fileapi_legacy = fileapi; +} + +/************************************************************************/ +/* pj_ctx_get_fileapi() */ +/************************************************************************/ + +projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) + +{ + if (nullptr==ctx) + return nullptr; + return ctx->fileapi_legacy; +} + diff --git a/src/filemanager.cpp b/src/filemanager.cpp new file mode 100644 index 00000000..b6164519 --- /dev/null +++ b/src/filemanager.cpp @@ -0,0 +1,173 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "filemanager.hpp" +#include "proj_internal.h" + +NS_PROJ_START + +// --------------------------------------------------------------------------- + +File::File() = default; + +// --------------------------------------------------------------------------- + +File::~File() = default; + +// --------------------------------------------------------------------------- + +class FileStdio : public File { + PJ_CONTEXT *m_ctx; + FILE *m_fp; + + FileStdio(const FileStdio &) = delete; + FileStdio &operator=(const FileStdio &) = delete; + + protected: + FileStdio(PJ_CONTEXT *ctx, FILE *fp) : m_ctx(ctx), m_fp(fp) {} + + public: + ~FileStdio() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileStdio::~FileStdio() { fclose(m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileStdio::read(void *buffer, size_t sizeBytes) { + return fread(buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileStdio::seek(unsigned long long offset, int whence) { + // TODO one day: use 64-bit offset compatible API + if (offset != static_cast(static_cast(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return fseek(m_fp, static_cast(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileStdio::tell() { + // TODO one day: use 64-bit offset compatible API + return ftell(m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr FileStdio::open(PJ_CONTEXT *ctx, const char *filename) { + auto fp = fopen(filename, "rb"); + return std::unique_ptr(fp ? new FileStdio(ctx, fp) : nullptr); +} + +// --------------------------------------------------------------------------- + +#ifndef REMOVE_LEGACY_SUPPORT + +class FileLegacyAdapter : public File { + PJ_CONTEXT *m_ctx; + PAFile m_fp; + + FileLegacyAdapter(const FileLegacyAdapter &) = delete; + FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; + + protected: + FileLegacyAdapter(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + + public: + ~FileLegacyAdapter() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileLegacyAdapter::~FileLegacyAdapter() { pj_ctx_fclose(m_ctx, m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileLegacyAdapter::read(void *buffer, size_t sizeBytes) { + return pj_ctx_fread(m_ctx, buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileLegacyAdapter::seek(unsigned long long offset, int whence) { + if (offset != static_cast(static_cast(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return pj_ctx_fseek(m_ctx, m_fp, static_cast(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileLegacyAdapter::tell() { + return pj_ctx_ftell(m_ctx, m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, + const char *filename) { + auto fid = pj_ctx_fopen(ctx, filename, "rb"); + return std::unique_ptr(fid ? new FileLegacyAdapter(ctx, fid) + : nullptr); +} + +#endif // REMOVE_LEGACY_SUPPORT + +// --------------------------------------------------------------------------- + +std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { +#ifndef REMOVE_LEGACY_SUPPORT + // If the user has specified a legacy fileapi, use it + if (ctx->fileapi_legacy != pj_get_default_fileapi()) { + return FileLegacyAdapter::open(ctx, filename); + } +#endif + return FileStdio::open(ctx, filename); +} + +NS_PROJ_END diff --git a/src/filemanager.hpp b/src/filemanager.hpp new file mode 100644 index 00000000..19478c48 --- /dev/null +++ b/src/filemanager.hpp @@ -0,0 +1,68 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FILEMANAGER_HPP_INCLUDED +#define FILEMANAGER_HPP_INCLUDED + +#include + +#include "proj.h" +#include "proj/util.hpp" + +NS_PROJ_START + +class File; + +class FileManager { + private: + FileManager() = delete; + + public: + // "Low-level" interface. + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); + + // "High-level" interface, honoring PROJ_LIB and the like. + static std::unique_ptr open_resource_file(PJ_CONTEXT *ctx, + const char *name); +}; + +// --------------------------------------------------------------------------- + +class File { + protected: + File(); + + public: + virtual ~File(); + virtual size_t read(void *buffer, size_t sizeBytes) = 0; + virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; + virtual unsigned long long tell() = 0; +}; + +NS_PROJ_END + +#endif // FILEMANAGER_HPP_INCLUDED \ No newline at end of file diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index d4c95092..eacc7a23 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -282,6 +282,8 @@ set(SRC_LIBPROJ_CORE tracing.cpp grids.hpp grids.cpp + filemanager.hpp + filemanager.cpp ${CMAKE_CURRENT_BINARY_DIR}/proj_config.h ) diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 24c31033..e1572754 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -44,6 +44,7 @@ #include "proj/internal/internal.hpp" #include "proj_internal.h" +#include "filemanager.hpp" static const char * proj_lib_name = #ifdef PROJ_LIB @@ -197,16 +198,17 @@ static const char *get_path_from_win32_projlib(const char *name, std::string& ou #endif /************************************************************************/ -/* pj_open_lib_ex() */ +/* pj_open_lib_internal() */ /************************************************************************/ -static PAFile -pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, - char* out_full_filename, size_t out_full_filename_size) { +static void* +pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, + void* (*open_file)(projCtx, const char*, const char*), + char* out_full_filename, size_t out_full_filename_size) { try { std::string fname; const char *sysname = nullptr; - PAFile fid = nullptr; + void* fid = nullptr; #ifdef WIN32 static const char dir_chars[] = "/\\"; const char dirSeparator = ';'; @@ -254,7 +256,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, fname += DIR_CHAR; fname += name; sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); + fid = open_file(ctx, sysname, mode); } catch( const std::exception& ) { } @@ -270,7 +272,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, fname += DIR_CHAR; fname += name; sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); + fid = open_file(ctx, sysname, mode); if( fid ) break; } @@ -290,7 +292,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, } assert(sysname); // to make Coverity Scan happy - if ( fid != nullptr || (fid = pj_ctx_fopen(ctx, sysname, mode)) != nullptr) + if ( fid != nullptr || (fid = open_file(ctx, sysname, mode)) != nullptr) { if( out_full_filename != nullptr && out_full_filename_size > 0 ) { @@ -321,19 +323,55 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, } } +/************************************************************************/ +/* pj_open_file_with_manager() */ +/************************************************************************/ + +static void* pj_open_file_with_manager(projCtx ctx, const char *name, + const char * /* mode */) +{ + return NS_PROJ::FileManager::open(ctx, name).release(); +} + +/************************************************************************/ +/* FileManager::open_resource_file() */ +/************************************************************************/ + +std::unique_ptr NS_PROJ::FileManager::open_resource_file( + projCtx ctx, const char *name) +{ + return std::unique_ptr( + reinterpret_cast( + pj_open_lib_internal(ctx, name, "rb", + pj_open_file_with_manager, + nullptr, 0))); +} + /************************************************************************/ /* pj_open_lib() */ /************************************************************************/ +#ifndef REMOVE_LEGACY_SUPPORT + +// Used by following legacy function +static void* pj_ctx_fopen_adapter(projCtx ctx, const char *name, const char *mode) +{ + return pj_ctx_fopen(ctx, name, mode); +} + +// Legacy function PAFile pj_open_lib(projCtx ctx, const char *name, const char *mode) { - return pj_open_lib_ex(ctx, name, mode, nullptr, 0); + return (PAFile)pj_open_lib_internal(ctx, name, mode, pj_ctx_fopen_adapter, nullptr, 0); } +#endif // REMOVE_LEGACY_SUPPORT + /************************************************************************/ /* pj_find_file() */ /************************************************************************/ + /** Returns the full filename corresponding to a proj resource file specified * as a short filename. * @@ -348,11 +386,14 @@ pj_open_lib(projCtx ctx, const char *name, const char *mode) { int pj_find_file(projCtx ctx, const char *short_filename, char* out_full_filename, size_t out_full_filename_size) { - PAFile f = pj_open_lib_ex(ctx, short_filename, "rb", out_full_filename, - out_full_filename_size); + auto f = reinterpret_cast( + pj_open_lib_internal(ctx, short_filename, "rb", + pj_open_file_with_manager, + out_full_filename, + out_full_filename_size)); if( f != nullptr ) { - pj_ctx_fclose(ctx, f); + delete f; return 1; } return 0; -- cgit v1.2.3 From cde119a2512341c07f775a8ee8f49c6920ec74f0 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 18 Dec 2019 23:53:09 +0100 Subject: grids.cpp: use FileManager/File interfaces --- src/grids.cpp | 241 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 115 insertions(+), 126 deletions(-) diff --git a/src/grids.cpp b/src/grids.cpp index 8fe6d745..91e51016 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -32,6 +32,7 @@ #define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include "grids.hpp" +#include "filemanager.hpp" #include "proj/internal/internal.hpp" #include "proj/internal/lru_cache.hpp" #include "proj_internal.h" @@ -151,41 +152,42 @@ bool NullVerticalShiftGrid::valueAt(int, int, float &out) const { class GTXVerticalShiftGrid : public VerticalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; GTXVerticalShiftGrid(const GTXVerticalShiftGrid &) = delete; GTXVerticalShiftGrid &operator=(const GTXVerticalShiftGrid &) = delete; public: - GTXVerticalShiftGrid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, - int widthIn, int heightIn, - const ExtentAndRes &extentIn) + explicit GTXVerticalShiftGrid(PJ_CONTEXT *ctx, std::unique_ptr &&fp, + const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) : VerticalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + m_fp(std::move(fp)) {} ~GTXVerticalShiftGrid() override; bool valueAt(int x, int y, float &out) const override; bool isNodata(float val, double multiplier) const override; - static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, PAFile fp, + static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &name); }; // --------------------------------------------------------------------------- -GTXVerticalShiftGrid::~GTXVerticalShiftGrid() { pj_ctx_fclose(m_ctx, m_fp); } +GTXVerticalShiftGrid::~GTXVerticalShiftGrid() = default; // --------------------------------------------------------------------------- -GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, +GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, + std::unique_ptr fp, const std::string &name) { unsigned char header[40]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -235,7 +237,8 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, extent.eastLon = (xorigin + xstep * (columns - 1)) * DEG_TO_RAD; extent.northLat = (yorigin + ystep * (rows - 1)) * DEG_TO_RAD; - return new GTXVerticalShiftGrid(ctx, fp, name, columns, rows, extent); + return new GTXVerticalShiftGrid(ctx, std::move(fp), name, columns, rows, + extent); } // --------------------------------------------------------------------------- @@ -243,8 +246,8 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp, bool GTXVerticalShiftGrid::valueAt(int x, int y, float &out) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); - pj_ctx_fseek(m_ctx, m_fp, 40 + sizeof(float) * (y * m_width + x), SEEK_SET); - if (pj_ctx_fread(m_ctx, &out, sizeof(out), 1, m_fp) != 1) { + m_fp->seek(40 + sizeof(float) * (y * m_width + x)); + if (m_fp->read(&out, sizeof(out)) != sizeof(out)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -679,7 +682,7 @@ std::string GTiffGrid::metadataItem(const std::string &key, int sample) const { class GTiffDataset { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; TIFF *m_hTIFF = nullptr; bool m_hasNextGrid = false; uint32 m_ifdIdx = 0; @@ -693,7 +696,7 @@ class GTiffDataset { // libtiff I/O routines static tsize_t tiffReadProc(thandle_t fd, tdata_t buf, tsize_t size) { GTiffDataset *self = static_cast(fd); - return pj_ctx_fread(self->m_ctx, buf, 1, size, self->m_fp); + return self->m_fp->read(buf, size); } static tsize_t tiffWriteProc(thandle_t, tdata_t, tsize_t) { @@ -703,11 +706,8 @@ class GTiffDataset { static toff_t tiffSeekProc(thandle_t fd, toff_t off, int whence) { GTiffDataset *self = static_cast(fd); - // FIXME Remove cast to long when pj_ctx_fseek supports unsigned long - // long - if (pj_ctx_fseek(self->m_ctx, self->m_fp, static_cast(off), - whence) == 0) - return static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); + if (self->m_fp->seek(off, whence)) + return static_cast(self->m_fp->tell()); else return static_cast(-1); } @@ -719,11 +719,10 @@ class GTiffDataset { static toff_t tiffSizeProc(thandle_t fd) { GTiffDataset *self = static_cast(fd); - const auto old_off = pj_ctx_ftell(self->m_ctx, self->m_fp); - pj_ctx_fseek(self->m_ctx, self->m_fp, 0, SEEK_END); - const auto file_size = - static_cast(pj_ctx_ftell(self->m_ctx, self->m_fp)); - pj_ctx_fseek(self->m_ctx, self->m_fp, old_off, SEEK_SET); + const auto old_off = self->m_fp->tell(); + self->m_fp->seek(0, SEEK_END); + const auto file_size = static_cast(self->m_fp->tell()); + self->m_fp->seek(old_off); return file_size; } @@ -732,7 +731,8 @@ class GTiffDataset { static void tiffUnmapProc(thandle_t, tdata_t, toff_t) {} public: - GTiffDataset(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + GTiffDataset(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : m_ctx(ctx), m_fp(std::move(fp)) {} virtual ~GTiffDataset(); bool openTIFF(const std::string &filename); @@ -745,7 +745,6 @@ class GTiffDataset { GTiffDataset::~GTiffDataset() { if (m_hTIFF) TIFFClose(m_hTIFF); - pj_ctx_fclose(m_ctx, m_fp); } // --------------------------------------------------------------------------- @@ -1048,13 +1047,15 @@ std::unique_ptr GTiffDataset::nextGrid() { class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { - GTiffVGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + GTiffVGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : GTiffDataset(ctx, std::move(fp)) {} public: ~GTiffVGridShiftSet() override; static std::unique_ptr - open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); + open(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &filename); }; #endif // TIFF_ENABLED @@ -1192,10 +1193,10 @@ void GTiffVGrid::insertGrid(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- std::unique_ptr -GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, +GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { - auto set = - std::unique_ptr(new GTiffVGridShiftSet(ctx, fp)); + auto set = std::unique_ptr( + new GTiffVGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; if (!set->openTIFF(filename)) { @@ -1293,15 +1294,14 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } if (ends_with(filename, "gtx") || ends_with(filename, "GTX")) { - auto grid = GTXVerticalShiftGrid::open(ctx, fp, filename); + auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), filename); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = @@ -1316,30 +1316,26 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { /* Load a header, to determine the file type. */ /* -------------------------------------------------------------------- */ unsigned char header[4]; - size_t header_size; - if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != - sizeof(header)) { - pj_ctx_fclose(ctx, fp); + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { return nullptr; } - pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + fp->seek(0); if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffVGridShiftSet::open(ctx, fp, filename); + auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); - pj_ctx_fclose(ctx, fp); return nullptr; #endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } @@ -1428,39 +1424,40 @@ static double to_double(const void *data) { class NTv1Grid : public HorizontalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; NTv1Grid(const NTv1Grid &) = delete; NTv1Grid &operator=(const NTv1Grid &) = delete; public: - NTv1Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, int widthIn, - int heightIn, const ExtentAndRes &extentIn) + explicit NTv1Grid(PJ_CONTEXT *ctx, std::unique_ptr &&fp, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn) : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + m_fp(std::move(fp)) {} ~NTv1Grid() override; bool valueAt(int, int, float &lonShift, float &latShift) const override; - static NTv1Grid *open(PJ_CONTEXT *ctx, PAFile fp, + static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); }; // --------------------------------------------------------------------------- -NTv1Grid::~NTv1Grid() { pj_ctx_fclose(m_ctx, m_fp); } +NTv1Grid::~NTv1Grid() = default; // --------------------------------------------------------------------------- -NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, +NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { unsigned char header[192]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1509,7 +1506,7 @@ NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, const int rows = static_cast( fabs((extent.northLat - extent.southLat) / extent.resLat + 0.5) + 1); - return new NTv1Grid(ctx, fp, filename, columns, rows, extent); + return new NTv1Grid(ctx, std::move(fp), filename, columns, rows, extent); } // --------------------------------------------------------------------------- @@ -1519,11 +1516,9 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { double two_doubles[2]; // NTv1 is organized from east to west ! - pj_ctx_fseek(m_ctx, m_fp, - 192 + 2 * sizeof(double) * (y * m_width + m_width - 1 - x), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_doubles[0], sizeof(two_doubles), 1, m_fp) != - 1) { + m_fp->seek(192 + 2 * sizeof(double) * (y * m_width + m_width - 1 - x)); + if (m_fp->read(&two_doubles[0], sizeof(two_doubles)) != + sizeof(two_doubles)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -1542,39 +1537,40 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { class CTable2Grid : public HorizontalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; CTable2Grid(const CTable2Grid &) = delete; CTable2Grid &operator=(const CTable2Grid &) = delete; public: - CTable2Grid(PJ_CONTEXT *ctx, PAFile fp, const std::string &nameIn, - int widthIn, int heightIn, const ExtentAndRes &extentIn) + CTable2Grid(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn) : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + m_fp(std::move(fp)) {} ~CTable2Grid() override; bool valueAt(int, int, float &lonShift, float &latShift) const override; - static CTable2Grid *open(PJ_CONTEXT *ctx, PAFile fp, + static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); }; // --------------------------------------------------------------------------- -CTable2Grid::~CTable2Grid() { pj_ctx_fclose(m_ctx, m_fp); } +CTable2Grid::~CTable2Grid() = default; // --------------------------------------------------------------------------- -CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, +CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { unsigned char header[160]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1615,7 +1611,7 @@ CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, extent.eastLon = extent.westLon + (width - 1) * extent.resLon; extent.northLat = extent.southLat + (height - 1) * extent.resLon; - return new CTable2Grid(ctx, fp, filename, width, height, extent); + return new CTable2Grid(ctx, std::move(fp), filename, width, height, extent); } // --------------------------------------------------------------------------- @@ -1625,9 +1621,8 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_floats[2]; - pj_ctx_fseek(m_ctx, m_fp, 160 + 2 * sizeof(float) * (y * m_width + x), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_floats[0], sizeof(two_floats), 1, m_fp) != 1) { + m_fp->seek(160 + 2 * sizeof(float) * (y * m_width + x)); + if (m_fp->read(&two_floats[0], sizeof(two_floats)) != sizeof(two_floats)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -1645,18 +1640,18 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, // --------------------------------------------------------------------------- class NTv2GridSet : public HorizontalShiftGridSet { - PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr m_fp; NTv2GridSet(const NTv2GridSet &) = delete; NTv2GridSet &operator=(const NTv2GridSet &) = delete; - NTv2GridSet(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + explicit NTv2GridSet(std::unique_ptr &&fp) : m_fp(std::move(fp)) {} public: ~NTv2GridSet() override; - static std::unique_ptr open(PJ_CONTEXT *ctx, PAFile fp, + static std::unique_ptr open(PJ_CONTEXT *ctx, + std::unique_ptr fp, const std::string &filename); }; @@ -1667,7 +1662,7 @@ class NTv2Grid : public HorizontalShiftGrid { std::string m_name; PJ_CONTEXT *m_ctx; // owned by the parent NTv2GridSet - PAFile m_fp; // owned by the parent NTv2GridSet + File *m_fp; // owned by the parent NTv2GridSet unsigned long long m_offset; bool m_mustSwap; @@ -1675,7 +1670,7 @@ class NTv2Grid : public HorizontalShiftGrid { NTv2Grid &operator=(const NTv2Grid &) = delete; public: - NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, PAFile fp, + NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, File *fp, unsigned long long offsetIn, bool mustSwapIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), @@ -1693,13 +1688,11 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { float two_float[2]; // NTv2 is organized from east to west ! // there are 4 components: lat shift, lon shift, lat error, lon error - pj_ctx_fseek( - m_ctx, m_fp, - // FIXME when fseek support unsigned long long - static_cast(m_offset + - 4 * sizeof(float) * (y * m_width + m_width - 1 - x)), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_float[0], sizeof(two_float), 1, m_fp) != 1) { + m_fp->seek( + m_offset + + 4 * sizeof(float) * + (static_cast(y) * m_width + m_width - 1 - x)); + if (m_fp->read(&two_float[0], sizeof(two_float)) != sizeof(two_float)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -1715,13 +1708,15 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { // --------------------------------------------------------------------------- -NTv2GridSet::~NTv2GridSet() { pj_ctx_fclose(m_ctx, m_fp); } +NTv2GridSet::~NTv2GridSet() = default; // --------------------------------------------------------------------------- -std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, +std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, + std::unique_ptr fp, const std::string &filename) { - auto set = std::unique_ptr(new NTv2GridSet(ctx, fp)); + File *fpRaw = fp.get(); + auto set = std::unique_ptr(new NTv2GridSet(std::move(fp))); set->m_name = filename; set->m_format = "ntv2"; @@ -1730,7 +1725,7 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1759,7 +1754,7 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, /* ==================================================================== */ for (unsigned subfile = 0; subfile < num_subfiles; subfile++) { // Read header - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -1824,10 +1819,10 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, return nullptr; } - auto offset = pj_ctx_ftell(ctx, fp); + const auto offset = fpRaw->tell(); auto grid = std::unique_ptr( - new NTv2Grid(filename + ", " + gridName, ctx, fp, offset, must_swap, - columns, rows, extent)); + new NTv2Grid(filename + ", " + gridName, ctx, fpRaw, offset, + must_swap, columns, rows, extent)); std::string parentName; parentName.assign(header + 24, 8); auto iter = mapGrids.find(parentName); @@ -1840,7 +1835,8 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, mapGrids[gridName] = gridPtr; // Skip grid data. 4 components of size float - pj_ctx_fseek(ctx, fp, gs_count * 4 * 4, SEEK_CUR); + fpRaw->seek(static_cast(gs_count) * 4 * 4, + SEEK_CUR); } return set; } @@ -1851,13 +1847,15 @@ std::unique_ptr NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { - GTiffHGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) : GTiffDataset(ctx, fp) {} + GTiffHGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : GTiffDataset(ctx, std::move(fp)) {} public: ~GTiffHGridShiftSet() override; static std::unique_ptr - open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); + open(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &filename); }; // --------------------------------------------------------------------------- @@ -1948,10 +1946,10 @@ void GTiffHGrid::insertGrid(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- std::unique_ptr -GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, +GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { - auto set = - std::unique_ptr(new GTiffHGridShiftSet(ctx, fp)); + auto set = std::unique_ptr( + new GTiffHGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; if (!set->openTIFF(filename)) { @@ -2127,8 +2125,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } @@ -2137,17 +2135,15 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { /* -------------------------------------------------------------------- */ /* Load a header, to determine the file type. */ /* -------------------------------------------------------------------- */ - size_t header_size; - if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != - sizeof(header)) { + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { /* some files may be smaller that sizeof(header), eg 160, so */ ctx->last_errno = 0; /* don't treat as a persistent error */ pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "pj_gridinfo_init: short header read of %d bytes", (int)header_size); } - - pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + fp->seek(0); /* -------------------------------------------------------------------- */ /* Determine file type. */ @@ -2155,9 +2151,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (header_size >= 144 + 16 && strncmp(header + 0, "HEADER", 6) == 0 && strncmp(header + 96, "W GRID", 6) == 0 && strncmp(header + 144, "TO NAD83 ", 16) == 0) { - auto grid = NTv1Grid::open(ctx, fp, filename); + auto grid = NTv1Grid::open(ctx, std::move(fp), filename); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = std::unique_ptr( @@ -2167,9 +2162,8 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { set->m_grids.push_back(std::unique_ptr(grid)); return set; } else if (header_size >= 9 && strncmp(header + 0, "CTABLE V2", 9) == 0) { - auto grid = CTable2Grid::open(ctx, fp, filename); + auto grid = CTable2Grid::open(ctx, std::move(fp), filename); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = std::unique_ptr( @@ -2181,24 +2175,22 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { } else if (header_size >= 48 + 7 && strncmp(header + 0, "NUM_OREC", 8) == 0 && strncmp(header + 48, "GS_TYPE", 7) == 0) { - return NTv2GridSet::open(ctx, fp, filename); + return NTv2GridSet::open(ctx, std::move(fp), filename); } else if (IsTIFF(header_size, reinterpret_cast(header))) { #ifdef TIFF_ENABLED - auto set = GTiffHGridShiftSet::open(ctx, fp, filename); + auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); - pj_ctx_fclose(ctx, fp); return nullptr; #endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } @@ -2247,14 +2239,15 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, class GTiffGenericGridShiftSet : public GenericShiftGridSet, public GTiffDataset { - GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, PAFile fp) - : GTiffDataset(ctx, fp) {} + GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr &&fp) + : GTiffDataset(ctx, std::move(fp)) {} public: ~GTiffGenericGridShiftSet() override; static std::unique_ptr - open(PJ_CONTEXT *ctx, PAFile fp, const std::string &filename); + open(PJ_CONTEXT *ctx, std::unique_ptr fp, + const std::string &filename); }; // --------------------------------------------------------------------------- @@ -2375,10 +2368,10 @@ bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { #ifdef TIFF_ENABLED std::unique_ptr -GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, PAFile fp, +GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename) { auto set = std::unique_ptr( - new GTiffGenericGridShiftSet(ctx, fp)); + new GTiffGenericGridShiftSet(ctx, std::move(fp))); set->m_name = filename; set->m_format = "gtiff"; if (!set->openTIFF(filename)) { @@ -2452,8 +2445,8 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } @@ -2462,30 +2455,26 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { /* Load a header, to determine the file type. */ /* -------------------------------------------------------------------- */ unsigned char header[4]; - size_t header_size; - if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != - sizeof(header)) { - pj_ctx_fclose(ctx, fp); + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { return nullptr; } - pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + fp->seek(0); if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffGenericGridShiftSet::open(ctx, fp, filename); + auto set = GTiffGenericGridShiftSet::open(ctx, std::move(fp), filename); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; #else pj_log(ctx, PJ_LOG_ERROR, "TIFF grid, but TIFF support disabled in this build"); - pj_ctx_fclose(ctx, fp); return nullptr; #endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } -- cgit v1.2.3 From d6ae5289b603dae07e5204695a7735d86a9c1c1e Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 16:25:58 +0100 Subject: Build: add optional curl dependency --- CMakeLists.txt | 15 +++++++++++ appveyor.yml | 1 + configure.ac | 48 ++++++++++++++++++++++++++++++++++++ src/Makefile.am | 4 +-- src/lib_proj.cmake | 5 ++++ travis/csa/before_install.sh | 2 +- travis/linux_clang/before_install.sh | 2 +- travis/linux_gcc/before_install.sh | 2 +- travis/linux_gcc7/before_install.sh | 2 +- travis/mingw32/install.sh | 4 +-- 10 files changed, 77 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 97a0b90a..26659ad2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,21 @@ if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) add_definitions(-DTIFF_ENABLED) endif() +################################################################################ +# Check for curl +################################################################################ + +option(ENABLE_CURL "Enable Curl support" ON) +if(ENABLE_CURL) + find_package(CURL REQUIRED) + if(CURL_FOUND) + boost_report_value(CURL_FOUND) + else() + message(SEND_ERROR "curl dependency not found!") + endif() + add_definitions(-DCURL_ENABLED) +endif() + ################################################################################ # threading configuration ################################################################################ diff --git a/appveyor.yml b/appveyor.yml index fb934f3a..2511854f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,6 +28,7 @@ build_script: - cd .. - vcpkg install sqlite3:"%platform%"-windows - vcpkg install tiff:"%platform%"-windows + - vcpkg install curl:"%platform%"-windows - set SQLITE3_BIN=%APPVEYOR_BUILD_FOLDER%\sqlite3\bin - mkdir %SQLITE3_BIN% - ps: | diff --git a/configure.ac b/configure.ac index 4aff6d74..23358dcb 100644 --- a/configure.ac +++ b/configure.ac @@ -320,6 +320,54 @@ AC_SUBST(TIFF_CFLAGS,$TIFF_CFLAGS) AC_SUBST(TIFF_LIBS,$TIFF_LIBS) AC_SUBST(TIFF_ENABLED_FLAGS,$TIFF_ENABLED_FLAGS) +dnl --------------------------------------------------------------------------- +dnl Check for curl +dnl --------------------------------------------------------------------------- + +FOUND_CURL=no +CURL_CFLAGS= +CURL_LIB= + +AC_ARG_WITH(curl, + [ --with-curl[=ARG] Enable curl support (ARG=path to curl-config.)],,,) + +dnl Clear some cache variables +unset ac_cv_path_LIBCURL + +if test "`basename xx/$with_curl`" = "curl-config" ; then + LIBCURL_CONFIG="$with_curl" +elif test "$with_curl" = "no" ; then + LIBCURL_CONFIG=no +else + AC_PATH_PROG(LIBCURL_CONFIG, curl-config, not-found) +fi + +if test "$LIBCURL_CONFIG" = "not-found" ; then + AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl]) +elif test "$LIBCURL_CONFIG" != "no" ; then + + CURL_VERNUM=`$LIBCURL_CONFIG --vernum` + CURL_VER=`$LIBCURL_CONFIG --version | awk '{print $2}'` + + AC_MSG_RESULT([ found libcurl version $CURL_VER]) + + AC_CHECK_LIB(curl,curl_global_init,FOUND_CURL=yes,FOUND_CURL=no,`$LIBCURL_CONFIG --libs`) + + if test "$FOUND_CURL" = "no" ; then + AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl]) + fi + CURL_ENABLED_FLAGS=-DCURL_ENABLED +fi + +if test "$FOUND_CURL" = "yes" ; then + CURL_CFLAGS=`$LIBCURL_CONFIG --cflags` + CURL_LIBS=`$LIBCURL_CONFIG --libs` +fi + +AC_SUBST(CURL_CFLAGS,$CURL_CFLAGS) +AC_SUBST(CURL_LIBS,$CURL_LIBS) +AC_SUBST(CURL_ENABLED_FLAGS,$CURL_ENABLED_FLAGS) + dnl --------------------------------------------------------------------------- dnl Check for external Google Test dnl --------------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index f5cabe5e..afe4bcb7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ + -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ @@ -43,7 +43,7 @@ geodtest_LDADD = libproj.la lib_LTLIBRARIES = libproj.la libproj_la_LDFLAGS = -no-undefined -version-info 17:0:2 -libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ +libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ @CURL_LIBS@ libproj_la_SOURCES = \ pj_list.h proj_internal.h \ diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index eacc7a23..12dcb366 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -446,6 +446,11 @@ if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) target_link_libraries(${PROJ_CORE_TARGET} ${TIFF_LIBRARY}) endif() +if(CURL_FOUND) + include_directories(${CURL_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${CURL_LIBRARY}) +endif() + if(MSVC) target_compile_definitions(${PROJ_CORE_TARGET} PRIVATE PROJ_MSVC_DLL_EXPORT=1) diff --git a/travis/csa/before_install.sh b/travis/csa/before_install.sh index 8d0f3fd2..dcb7733b 100755 --- a/travis/csa/before_install.sh +++ b/travis/csa/before_install.sh @@ -5,7 +5,7 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev CLANG_LLVM=clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-16.04 wget http://releases.llvm.org/6.0.0/$CLANG_LLVM.tar.xz diff --git a/travis/linux_clang/before_install.sh b/travis/linux_clang/before_install.sh index cfe7ba67..c4b8acad 100755 --- a/travis/linux_clang/before_install.sh +++ b/travis/linux_clang/before_install.sh @@ -5,4 +5,4 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev diff --git a/travis/linux_gcc/before_install.sh b/travis/linux_gcc/before_install.sh index d4deb85e..9db2ea89 100755 --- a/travis/linux_gcc/before_install.sh +++ b/travis/linux_gcc/before_install.sh @@ -9,7 +9,7 @@ sudo apt-get install -qq \ lcov \ doxygen graphviz \ sqlite3 libsqlite3-dev \ - libtiff-dev \ + libtiff-dev libcurl4-openssl-dev \ cppcheck scripts/cppcheck.sh diff --git a/travis/linux_gcc7/before_install.sh b/travis/linux_gcc7/before_install.sh index b3e053c0..2ef2feee 100755 --- a/travis/linux_gcc7/before_install.sh +++ b/travis/linux_gcc7/before_install.sh @@ -9,7 +9,7 @@ sudo apt-get install -qq \ lcov \ doxygen graphviz \ sqlite3 libsqlite3-dev \ - libtiff-dev + libtiff-dev libcurl4-openssl-dev #scripts/cppcheck.sh #scripts/doxygen.sh diff --git a/travis/mingw32/install.sh b/travis/mingw32/install.sh index bc9f3258..abe79c7b 100755 --- a/travis/mingw32/install.sh +++ b/travis/mingw32/install.sh @@ -47,7 +47,7 @@ ln -s /usr/$MINGW_ARCH/bin/libsqlite3-0.dll $WINE_SYSDIR # autoconf build mkdir build_autoconf cd build_autoconf -CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install +CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install --without-curl make -j2 make install make dist-all @@ -63,7 +63,7 @@ cd .. # autoconf build with grids mkdir build_autoconf_grids cd build_autoconf_grids -CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install_nad +CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install_nad --without-curl make -j2 make install find /tmp/proj_autoconf_install_nad -- cgit v1.2.3 From 2aeb122fb2720fa6809b9429aca5154f8eec5e6d Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 16 Dec 2019 22:13:32 +0100 Subject: travis/osx/before_install.sh: do not install md5sha1sum from brew --- travis/osx/before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/osx/before_install.sh b/travis/osx/before_install.sh index 5de816c8..4a52cd30 100755 --- a/travis/osx/before_install.sh +++ b/travis/osx/before_install.sh @@ -9,7 +9,7 @@ brew install ccache #brew upgrade sqlite3 #brew upgrade libtiff brew install doxygen -brew install md5sha1sum +#brew install md5sha1sum brew reinstall python brew reinstall wget -- cgit v1.2.3 From a06f4a258f618dbad2ce01feadab6908db00bda5 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 21:24:53 +0100 Subject: Add proj_context_set_network_callbacks() with an empty implementation --- Doxyfile | 2 +- scripts/doxygen.sh | 2 +- scripts/reference_exported_symbols.txt | 1 + src/ctx.cpp | 3 + src/filemanager.cpp | 164 +++++++++++++++++++++++++++++++++ src/filemanager.hpp | 6 ++ src/open_lib.cpp | 6 +- src/proj.h | 76 +++++++++++++++ src/proj_internal.h | 13 +++ 9 files changed, 270 insertions(+), 3 deletions(-) diff --git a/Doxyfile b/Doxyfile index da9a6b61..2d263ed7 100644 --- a/Doxyfile +++ b/Doxyfile @@ -772,7 +772,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox +INPUT = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox src/filemanager.cpp # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/scripts/doxygen.sh b/scripts/doxygen.sh index 48646237..9646eed8 100755 --- a/scripts/doxygen.sh +++ b/scripts/doxygen.sh @@ -34,7 +34,7 @@ fi mkdir -p docs/build/tmp_breathe python scripts/generate_breathe_friendly_general_doc.py rm -rf docs/build/xml/ -(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj src/proj.h docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen - > docs/build/tmp_breathe/docs_log.txt 2>&1 +(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj src/proj.h src/filemanager.cpp docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen - > docs/build/tmp_breathe/docs_log.txt 2>&1 if grep -i warning docs/build/tmp_breathe/docs_log.txt; then echo "Doxygen warnings found" && cat docs/build/tmp_breathe/docs_log.txt && /bin/false; else diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index e20f29b3..6e4e5302 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -795,6 +795,7 @@ proj_context_guess_wkt_dialect proj_context_set_autoclose_database proj_context_set_database_path proj_context_set_file_finder +proj_context_set_network_callbacks proj_context_set(PJconsts*, projCtx_t*) proj_context_set_search_paths proj_context_use_proj4_init_rules diff --git a/src/ctx.cpp b/src/ctx.cpp index da90d4d7..a7cf8cb3 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -33,6 +33,7 @@ #include "proj_experimental.h" #include "proj_internal.h" +#include "filemanager.hpp" /************************************************************************/ /* pj_get_ctx() */ @@ -96,6 +97,7 @@ projCtx_t projCtx_t::createDefault() ctx.debug_level = PJ_LOG_NONE; ctx.logger = pj_stderr_logger; ctx.fileapi_legacy = pj_get_default_fileapi(); + NS_PROJ::FileManager::fillDefaultNetworkInterface(&ctx); if( getenv("PROJ_DEBUG") != nullptr ) { @@ -139,6 +141,7 @@ projCtx_t::projCtx_t(const projCtx_t& other) file_finder = other.file_finder; file_finder_legacy = other.file_finder_legacy; file_finder_user_data = other.file_finder_user_data; + networking = other.networking; } /************************************************************************/ diff --git a/src/filemanager.cpp b/src/filemanager.cpp index b6164519..97f6a4dc 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -25,11 +25,21 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ +#ifndef FROM_PROJ_CPP +#define FROM_PROJ_CPP +#endif + #include "filemanager.hpp" +#include "proj.h" +#include "proj/internal/internal.hpp" #include "proj_internal.h" +//! @cond Doxygen_Suppress + NS_PROJ_START +using namespace internal; + // --------------------------------------------------------------------------- File::File() = default; @@ -160,6 +170,79 @@ std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +class NetworkFile : public File { + PJ_CONTEXT *m_ctx; + PROJ_NETWORK_HANDLE *m_handle; + unsigned long long m_pos = 0; + + NetworkFile(const NetworkFile &) = delete; + NetworkFile &operator=(const NetworkFile &) = delete; + + protected: + NetworkFile(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle) + : m_ctx(ctx), m_handle(handle) {} + + public: + ~NetworkFile() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence) override; + unsigned long long tell() override; + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { + std::vector buffer(16 * 1024); + size_t size_read = 0; + auto handle = ctx->networking.open(ctx, filename, buffer.size(), &buffer[0], + &size_read, ctx->networking.user_data); + return std::unique_ptr(handle ? new NetworkFile(ctx, handle) + : nullptr); +} + +// --------------------------------------------------------------------------- + +size_t NetworkFile::read(void *buffer, size_t sizeBytes) { + size_t nRead = m_ctx->networking.read_range( + m_ctx, m_handle, m_pos, sizeBytes, buffer, m_ctx->networking.user_data); + m_pos += nRead; + return nRead; +} + +// --------------------------------------------------------------------------- + +bool NetworkFile::seek(unsigned long long offset, int whence) { + if (whence == SEEK_SET) { + m_pos = offset; + } else if (whence == SEEK_CUR) { + m_pos += offset; + } else { + if (offset != 0) + return false; + const auto filesize = m_ctx->networking.get_file_size( + m_ctx, m_handle, m_ctx->networking.user_data); + if (filesize == 0) + return false; + m_pos = filesize; + } + return true; +} + +// --------------------------------------------------------------------------- + +unsigned long long NetworkFile::tell() { return m_pos; } + +// --------------------------------------------------------------------------- + +NetworkFile::~NetworkFile() { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); +} + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifndef REMOVE_LEGACY_SUPPORT // If the user has specified a legacy fileapi, use it @@ -167,7 +250,88 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { return FileLegacyAdapter::open(ctx, filename); } #endif + if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + return NetworkFile::open(ctx, filename); + } return FileStdio::open(ctx, filename); } +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE * +no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ + void * /*user_data*/) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); + return nullptr; +} + +// --------------------------------------------------------------------------- + +static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, + void * /*user_data*/) {} + +// --------------------------------------------------------------------------- + +static const char *no_op_network_get_last_error(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *, + void * /*user_data*/) { + return "Network functionality not available"; +} + +// --------------------------------------------------------------------------- + +void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { + ctx->networking.open = no_op_network_open; + ctx->networking.close = no_op_network_close; + ctx->networking.get_last_error = no_op_network_get_last_error; +} + +// --------------------------------------------------------------------------- + NS_PROJ_END + +//! @endcond + +// --------------------------------------------------------------------------- + +/** Define a custom set of callbacks for network access. + * + * All callbacks should be provided (non NULL pointers). + * + * @param ctx PROJ context, or NULL + * @param open_cbk Callback to open a remote file given its URL + * @param close_cbk Callback to close a remote file. + * @param get_header_value_cbk Callback to get HTTP headers + * @param get_file_size_cbk Callback to get the size of the remote file. + * @param read_range_cbk Callback to read a range of bytes inside a remote file. + * @param get_last_error_cbk Callback to get last error message. + * @param user_data Arbitrary pointer provided by the user, and passed to the + * above callbacks. May be NULL. + * @return TRUE in case of success. + */ +int proj_context_set_network_callbacks( + PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_get_file_size_cbk_type get_file_size_cbk, + proj_network_read_range_type read_range_cbk, + proj_network_get_last_error_type get_last_error_cbk, void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!open_cbk || !close_cbk || !get_header_value_cbk || + !get_file_size_cbk || !read_range_cbk || !get_last_error_cbk) { + return false; + } + ctx->networking.open = open_cbk; + ctx->networking.close = close_cbk; + ctx->networking.get_header_value = get_header_value_cbk; + ctx->networking.get_file_size = get_file_size_cbk; + ctx->networking.read_range = read_range_cbk; + ctx->networking.get_last_error = get_last_error_cbk; + ctx->networking.user_data = user_data; + return true; +} diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 19478c48..0e830450 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -35,6 +35,8 @@ NS_PROJ_START +//! @cond Doxygen_Suppress + class File; class FileManager { @@ -48,6 +50,8 @@ class FileManager { // "High-level" interface, honoring PROJ_LIB and the like. static std::unique_ptr open_resource_file(PJ_CONTEXT *ctx, const char *name); + + static void fillDefaultNetworkInterface(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -63,6 +67,8 @@ class File { virtual unsigned long long tell() = 0; }; +//! @endcond Doxygen_Suppress + NS_PROJ_END #endif // FILEMANAGER_HPP_INCLUDED \ No newline at end of file diff --git a/src/open_lib.cpp b/src/open_lib.cpp index e1572754..6bdd5510 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -53,6 +53,8 @@ PROJ_LIB; nullptr; #endif +using namespace NS_PROJ::internal; + /************************************************************************/ /* pj_set_finder() */ /************************************************************************/ @@ -238,7 +240,9 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, else if (strchr(dir_chars,*name) || (*name == '.' && strchr(dir_chars,name[1])) || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) - || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) ) + || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) + || starts_with(name, "http://") + || starts_with(name, "https://")) sysname = name; /* or try to use application provided file finder */ diff --git a/src/proj.h b/src/proj.h index fc309542..ba6e7043 100644 --- a/src/proj.h +++ b/src/proj.h @@ -353,6 +353,82 @@ void PROJ_DLL proj_context_set_search_paths(PJ_CONTEXT *ctx, int count_paths, co void PROJ_DLL proj_context_use_proj4_init_rules(PJ_CONTEXT *ctx, int enable); int PROJ_DLL proj_context_get_use_proj4_init_rules(PJ_CONTEXT *ctx, int from_legacy_code_path); +/*! @endcond */ + +/** Opaque structure for PROJ. Implementations might cast it to their + * structure/class of choice. */ +typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; + +/** Network access: open callback + * + * Should try to read the size_to_read first bytes of the file of URL url, + * and write them to buffer. *out_size_read should be updated with the actual + * amount of bytes read (== size_to_read if the file is larger than size_to_read). + * During this read, the implementation should make sure to store the HTTP + * headers from the server response to be able to respond to + * proj_network_get_header_value_cbk_type and proj_network_get_file_size_cbk_type + * callbacks. + * + * @return a non-NULL opaque handle in case of success. + */ +typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( + PJ_CONTEXT* ctx, + const char* url, + size_t size_to_read, + void* buffer, + size_t* out_size_read, + void* user_data); + +/** Network access: close callback */ +typedef void (*proj_network_close_cbk_type)(PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + void* user_data); + +/** Network access: get HTTP headers */ +typedef const char* (*proj_network_get_header_value_cbk_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + const char* header_name, + void* user_data); + +/** Network access: get file size */ +typedef unsigned long long (*proj_network_get_file_size_cbk_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + void* user_data); + +/** Network access: read range + * + * Read size_to_read bytes from handle, starting at offset, into + * buffer. + * @return the number of bytes actually read (0 in case of error) + */ +typedef size_t (*proj_network_read_range_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + unsigned long long offset, + size_t size_to_read, + void* buffer, + void* user_data); + +/** Network access: get last error message */ +typedef const char* (*proj_network_get_last_error_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE*, + void* user_data); + +int PROJ_DLL proj_context_set_network_callbacks( + PJ_CONTEXT* ctx, + proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_get_file_size_cbk_type get_file_size_cbk, + proj_network_read_range_type read_range_cbk, + proj_network_get_last_error_type get_last_error_cbk, + void* user_data); + +/*! @cond Doxygen_Suppress */ + /* Manage the transformation definition object PJ */ PJ PROJ_DLL *proj_create (PJ_CONTEXT *ctx, const char *definition); PJ PROJ_DLL *proj_create_argv (PJ_CONTEXT *ctx, int argc, char **argv); diff --git a/src/proj_internal.h b/src/proj_internal.h index 93134946..aefeea39 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -666,6 +666,17 @@ struct projFileAPI_t; struct projCppContext; +struct projNetworkCallbacksAndData +{ + proj_network_open_cbk_type open = nullptr; + proj_network_close_cbk_type close = nullptr; + proj_network_get_header_value_cbk_type get_header_value = nullptr; + proj_network_get_file_size_cbk_type get_file_size = nullptr; + proj_network_read_range_type read_range = nullptr; + proj_network_get_last_error_type get_last_error = nullptr; + void* user_data = nullptr; +}; + /* proj thread context */ struct projCtx_t { int last_errno = 0; @@ -684,6 +695,8 @@ struct projCtx_t { const char* (*file_finder) (PJ_CONTEXT *, const char*, void* user_data) = nullptr; void* file_finder_user_data = nullptr; + projNetworkCallbacksAndData networking{}; + int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() projCtx_t() = default; -- cgit v1.2.3 From f73527fa20c9250cfced6aa73a7e46f40dc2c214 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Thu, 19 Dec 2019 22:36:07 +0100 Subject: Add very minimalistic and slow libcurl implementation --- CMakeLists.txt | 3 + configure.ac | 4 +- src/filemanager.cpp | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 26659ad2..2125b93a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,9 @@ elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") set(PROJ_CXX_WARN_FLAGS -Wall) endif() endif() + +add_definitions(-DNOMINMAX) + set(PROJ_C_WARN_FLAGS "${PROJ_C_WARN_FLAGS}" CACHE STRING "C flags used to compile PROJ targets") set(PROJ_CXX_WARN_FLAGS "${PROJ_CXX_WARN_FLAGS}" diff --git a/configure.ac b/configure.ac index 23358dcb..afde2712 100644 --- a/configure.ac +++ b/configure.ac @@ -168,8 +168,8 @@ AC_SUBST(C_WFLAGS,$C_WFLAGS) AC_SUBST(CXX_WFLAGS,$CXX_WFLAGS) AC_SUBST(NO_ZERO_AS_NULL_POINTER_CONSTANT_FLAG,$NO_ZERO_AS_NULL_POINTER_CONSTANT_FLAG) -CFLAGS="${CFLAGS} -fvisibility=hidden" -CXXFLAGS="${CXXFLAGS} -fvisibility=hidden" +CFLAGS="${CFLAGS} -fvisibility=hidden -DNOMINMAX" +CXXFLAGS="${CXXFLAGS} -fvisibility=hidden -DNOMINMAX" dnl Checks for libraries. save_CFLAGS="$CFLAGS" diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 97f6a4dc..3a2acee4 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -29,11 +29,18 @@ #define FROM_PROJ_CPP #endif +#include + #include "filemanager.hpp" #include "proj.h" #include "proj/internal/internal.hpp" #include "proj_internal.h" +#ifdef CURL_ENABLED +#include +#include // for sqlite3_snprintf +#endif + //! @cond Doxygen_Suppress NS_PROJ_START @@ -258,6 +265,152 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { // --------------------------------------------------------------------------- +#ifdef CURL_ENABLED + +struct CurlFileHandle { + CURL *m_handle = nullptr; + std::string m_headers; + + CurlFileHandle(const CurlFileHandle &) = delete; + CurlFileHandle &operator=(const CurlFileHandle &) = delete; + + explicit CurlFileHandle(CURL *handle, std::string &&headers) + : m_handle(handle), m_headers(std::move(headers)) {} + ~CurlFileHandle(); +}; + +// --------------------------------------------------------------------------- + +CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } + +// --------------------------------------------------------------------------- + +static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, + void *req) { + const size_t nSize = count * nmemb; + auto pStr = static_cast(req); + if (pStr->size() + nSize > pStr->capacity()) { + // to avoid servers not honouring Range to cause excessive memory + // allocation + return 0; + } + pStr->append(static_cast(buffer), nSize); + return nmemb; +} + +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, + size_t size_to_read, void *buffer, + size_t *out_size_read, void *) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return nullptr; + curl_easy_setopt(hCurlHandle, CURLOPT_URL, url); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(hCurlHandle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(hCurlHandle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(hCurlHandle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "0-%llu", size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string headers; + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, pj_curl_write_func); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, nullptr); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + curl_easy_cleanup(hCurlHandle); + return nullptr; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + *out_size_read = std::min(size_to_read, body.size()); + + return reinterpret_cast( + new CurlFileHandle(hCurlHandle, std::move(headers))); +} + +// --------------------------------------------------------------------------- + +static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, + void * /*user_data*/) { + delete reinterpret_cast(handle); +} + +// --------------------------------------------------------------------------- + +static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, + unsigned long long offset, size_t size_to_read, + void *buffer, void *) { + auto handle = reinterpret_cast(raw_handle); + auto hCurlHandle = handle->m_handle; + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + return 0; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + return std::min(size_to_read, body.size()); +} + +#else + +// --------------------------------------------------------------------------- + static PROJ_NETWORK_HANDLE * no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, size_t, /* size to read */ @@ -281,12 +434,20 @@ static const char *no_op_network_get_last_error(PJ_CONTEXT *, return "Network functionality not available"; } +#endif + // --------------------------------------------------------------------------- void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { +#ifdef CURL_ENABLED + ctx->networking.open = pj_curl_open; + ctx->networking.close = pj_curl_close; + ctx->networking.read_range = pj_curl_read_range; +#else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; ctx->networking.get_last_error = no_op_network_get_last_error; +#endif } // --------------------------------------------------------------------------- -- cgit v1.2.3 From ed92965937856ae697010e1461e82f6245d19909 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 20 Dec 2019 19:10:20 +0100 Subject: Network: add a memory cache and I/O chunking strategy --- src/filemanager.cpp | 259 ++++++++++++++++++++++++++++++++++++++++++++++++---- src/filemanager.hpp | 2 + src/malloc.cpp | 2 + src/proj.h | 16 ++-- 4 files changed, 256 insertions(+), 23 deletions(-) diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 3a2acee4..2dfcd6b6 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -28,14 +28,34 @@ #ifndef FROM_PROJ_CPP #define FROM_PROJ_CPP #endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include +#include +#include #include "filemanager.hpp" #include "proj.h" #include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" #include "proj_internal.h" +#ifdef __MINGW32__ +// mingw32-win32 doesn't implement std::mutex +namespace { +class MyMutex { + public: + // cppcheck-suppress functionStatic + void lock() { pj_acquire_lock(); } + // cppcheck-suppress functionStatic + void unlock() { pj_release_lock(); } +}; +} +#else +#include +#define MyMutex std::mutex +#endif + #ifdef CURL_ENABLED #include #include // for sqlite3_snprintf @@ -177,17 +197,97 @@ std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +constexpr size_t DOWNLOAD_CHUNK_SIZE = 16 * 1024; +constexpr int MAX_CHUNKS = 64; + +class NetworkChunkCache { + public: + void insert(const std::string &url, unsigned long long chunkIdx, + std::vector &&data); + + std::shared_ptr> + get(const std::string &url, unsigned long long chunkIdx); + + void clear(); + + private: + struct Key { + std::string url; + unsigned long long chunkIdx; + + Key(const std::string &urlIn, unsigned long long chunkIdxIn) + : url(urlIn), chunkIdx(chunkIdxIn) {} + bool operator==(const Key &other) const { + return url == other.url && chunkIdx == other.chunkIdx; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return std::hash{}(k.url) ^ + (std::hash{}(k.chunkIdx) << 1); + } + }; + + lru11::Cache< + Key, std::shared_ptr>, MyMutex, + std::unordered_map< + Key, + typename std::list>>>::iterator, + KeyHasher>> + cache_{MAX_CHUNKS}; +}; + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::insert(const std::string &url, + unsigned long long chunkIdx, + std::vector &&data) { + cache_.insert( + Key(url, chunkIdx), + std::make_shared>(std::move(data))); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr> +NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) { + std::shared_ptr> ret; + cache_.tryGet(Key(url, chunkIdx), ret); + return ret; +} + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::clear() +{ + cache_.clear(); +} + +// --------------------------------------------------------------------------- + +static NetworkChunkCache gNetworkChunkCache{}; + +// --------------------------------------------------------------------------- + class NetworkFile : public File { PJ_CONTEXT *m_ctx; + std::string m_url; PROJ_NETWORK_HANDLE *m_handle; unsigned long long m_pos = 0; + size_t m_nBlocksToDownload = 1; + unsigned long long m_lastDownloadedOffset; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; protected: - NetworkFile(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle) - : m_ctx(ctx), m_handle(handle) {} + NetworkFile(PJ_CONTEXT *ctx, const std::string &url, + PROJ_NETWORK_HANDLE *handle, + unsigned long long lastDownloadOffset) + : m_ctx(ctx), m_url(url), m_handle(handle), + m_lastDownloadedOffset(lastDownloadOffset) {} public: ~NetworkFile() override; @@ -202,20 +302,125 @@ class NetworkFile : public File { // --------------------------------------------------------------------------- std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { - std::vector buffer(16 * 1024); - size_t size_read = 0; - auto handle = ctx->networking.open(ctx, filename, buffer.size(), &buffer[0], - &size_read, ctx->networking.user_data); - return std::unique_ptr(handle ? new NetworkFile(ctx, handle) - : nullptr); + if (gNetworkChunkCache.get(filename, 0)) { + return std::unique_ptr( + new NetworkFile(ctx, filename, nullptr, + std::numeric_limits::max())); + } else { + std::vector buffer(DOWNLOAD_CHUNK_SIZE); + size_t size_read = 0; + auto handle = + ctx->networking.open(ctx, filename, 0, buffer.size(), &buffer[0], + &size_read, ctx->networking.user_data); + buffer.resize(size_read); + gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + return std::unique_ptr( + handle ? new NetworkFile(ctx, filename, handle, size_read) + : nullptr); + } } // --------------------------------------------------------------------------- size_t NetworkFile::read(void *buffer, size_t sizeBytes) { - size_t nRead = m_ctx->networking.read_range( - m_ctx, m_handle, m_pos, sizeBytes, buffer, m_ctx->networking.user_data); - m_pos += nRead; + + if (sizeBytes == 0) + return 0; + + auto iterOffset = m_pos; + while (sizeBytes) { + const auto chunkIdxToDownload = iterOffset / DOWNLOAD_CHUNK_SIZE; + const auto offsetToDownload = chunkIdxToDownload * DOWNLOAD_CHUNK_SIZE; + std::vector region; + auto pChunk = gNetworkChunkCache.get(m_url, chunkIdxToDownload); + if (pChunk != nullptr) { + region = *pChunk; + } else { + if (offsetToDownload == m_lastDownloadedOffset) { + // In case of consecutive reads (of small size), we use a + // heuristic that we will read the file sequentially, so + // we double the requested size to decrease the number of + // client/server roundtrips. + if (m_nBlocksToDownload < 100) + m_nBlocksToDownload *= 2; + } else { + // Random reads. Cancel the above heuristics. + m_nBlocksToDownload = 1; + } + + // Ensure that we will request at least the number of blocks + // to satisfy the remaining buffer size to read. + const auto endOffsetToDownload = + ((iterOffset + sizeBytes + DOWNLOAD_CHUNK_SIZE - 1) / + DOWNLOAD_CHUNK_SIZE) * + DOWNLOAD_CHUNK_SIZE; + const auto nMinBlocksToDownload = static_cast( + (endOffsetToDownload - offsetToDownload) / DOWNLOAD_CHUNK_SIZE); + if (m_nBlocksToDownload < nMinBlocksToDownload) + m_nBlocksToDownload = nMinBlocksToDownload; + + // Avoid reading already cached data. + // Note: this might get evicted if concurrent reads are done, but + // this should not cause bugs. Just missed optimization. + for (size_t i = 1; i < m_nBlocksToDownload; i++) { + if (gNetworkChunkCache.get(m_url, chunkIdxToDownload + i) != + nullptr) { + m_nBlocksToDownload = i; + break; + } + } + + if (m_nBlocksToDownload > MAX_CHUNKS) + m_nBlocksToDownload = MAX_CHUNKS; + + region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); + size_t nRead = 0; + if (!m_handle) { + m_handle = m_ctx->networking.open( + m_ctx, m_url.c_str(), offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + &nRead, m_ctx->networking.user_data); + if (!m_handle) { + return 0; + } + } else { + nRead = m_ctx->networking.read_range( + m_ctx, m_handle, offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + m_ctx->networking.user_data); + } + if (nRead == 0) { + return 0; + } + region.resize(nRead); + m_lastDownloadedOffset = offsetToDownload + nRead; + + const auto nChunks = + (region.size() + DOWNLOAD_CHUNK_SIZE - 1) / DOWNLOAD_CHUNK_SIZE; + for (size_t i = 0; i < nChunks; i++) { + std::vector chunk( + region.data() + i * DOWNLOAD_CHUNK_SIZE, + region.data() + + std::min((i + 1) * DOWNLOAD_CHUNK_SIZE, region.size())); + gNetworkChunkCache.insert(m_url, chunkIdxToDownload + i, + std::move(chunk)); + } + } + const size_t nToCopy = static_cast( + std::min(static_cast(sizeBytes), + region.size() - (iterOffset - offsetToDownload))); + memcpy(buffer, region.data() + iterOffset - offsetToDownload, nToCopy); + buffer = static_cast(buffer) + nToCopy; + iterOffset += nToCopy; + sizeBytes -= nToCopy; + if (region.size() < static_cast(DOWNLOAD_CHUNK_SIZE) && + sizeBytes != 0) { + break; + } + } + + size_t nRead = static_cast(iterOffset - m_pos); + m_pos = iterOffset; return nRead; } @@ -229,6 +434,16 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; + if (!m_handle) { + size_t nRead = 0; + char dummy; + m_handle = + m_ctx->networking.open(m_ctx, m_url.c_str(), 0, 1, &dummy, + &nRead, m_ctx->networking.user_data); + if (!m_handle) { + return false; + } + } const auto filesize = m_ctx->networking.get_file_size( m_ctx, m_handle, m_ctx->networking.user_data); if (filesize == 0) @@ -245,7 +460,9 @@ unsigned long long NetworkFile::tell() { return m_pos; } // --------------------------------------------------------------------------- NetworkFile::~NetworkFile() { - m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + if (m_handle) { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + } } // --------------------------------------------------------------------------- @@ -301,6 +518,7 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, + unsigned long long offset, size_t size_to_read, void *buffer, size_t *out_size_read, void *) { CURL *hCurlHandle = curl_easy_init(); @@ -326,7 +544,8 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, } char szBuffer[128]; - sqlite3_snprintf(sizeof(szBuffer), szBuffer, "0-%llu", size_to_read - 1); + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); std::string headers; @@ -413,9 +632,10 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, static PROJ_NETWORK_HANDLE * no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, - size_t, /* size to read */ - void *, /* buffer to update with bytes read*/ - size_t *, /* output: size actually read */ + unsigned long long, /* offset */ + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ void * /*user_data*/) { pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); return nullptr; @@ -452,6 +672,13 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- +void FileManager::clearCache() +{ + gNetworkChunkCache.clear(); +} + +// --------------------------------------------------------------------------- + NS_PROJ_END //! @endcond diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 0e830450..949b223f 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -52,6 +52,8 @@ class FileManager { const char *name); static void fillDefaultNetworkInterface(PJ_CONTEXT *ctx); + + static void clearCache(); }; // --------------------------------------------------------------------------- diff --git a/src/malloc.cpp b/src/malloc.cpp index 3fd3699f..1c539b6b 100644 --- a/src/malloc.cpp +++ b/src/malloc.cpp @@ -49,6 +49,7 @@ #include "proj.h" #include "proj_internal.h" #include "grids.hpp" +#include "filemanager.hpp" using namespace NS_PROJ; @@ -262,4 +263,5 @@ void proj_cleanup() { /*****************************************************************************/ pj_clear_initcache(); pj_deallocate_grids(); + FileManager::clearCache(); } diff --git a/src/proj.h b/src/proj.h index ba6e7043..5f080285 100644 --- a/src/proj.h +++ b/src/proj.h @@ -361,7 +361,8 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; /** Network access: open callback * - * Should try to read the size_to_read first bytes of the file of URL url, + * Should try to read the size_to_read first bytes at the specified offset of + * the file given by URL url, * and write them to buffer. *out_size_read should be updated with the actual * amount of bytes read (== size_to_read if the file is larger than size_to_read). * During this read, the implementation should make sure to store the HTTP @@ -372,12 +373,13 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; * @return a non-NULL opaque handle in case of success. */ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( - PJ_CONTEXT* ctx, - const char* url, - size_t size_to_read, - void* buffer, - size_t* out_size_read, - void* user_data); + PJ_CONTEXT* ctx, + const char* url, + unsigned long long offset, + size_t size_to_read, + void* buffer, + size_t* out_size_read, + void* user_data); /** Network access: close callback */ typedef void (*proj_network_close_cbk_type)(PJ_CONTEXT* ctx, -- cgit v1.2.3 From d3bdbb841719780560438129b4911a86a7a4be58 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 21 Dec 2019 00:38:50 +0100 Subject: Add testing of network functionality --- data/Makefile.am | 1 + data/tests/egm96_15_uncompressed_truncated.tif | Bin 0 -> 956 bytes src/filemanager.cpp | 10 +- test/unit/CMakeLists.txt | 15 + test/unit/Makefile.am | 10 +- test/unit/test_network.cpp | 497 +++++++++++++++++++++++++ 6 files changed, 524 insertions(+), 9 deletions(-) create mode 100644 data/tests/egm96_15_uncompressed_truncated.tif create mode 100644 test/unit/test_network.cpp diff --git a/data/Makefile.am b/data/Makefile.am index 2a8eb285..59ab4e83 100644 --- a/data/Makefile.am +++ b/data/Makefile.am @@ -80,6 +80,7 @@ EXTRA_DIST = GL27 nad.lst nad27 nad83 \ tests/test_hgrid_with_subgrid.tif \ tests/test_hgrid_with_subgrid_no_grid_name.tif \ tests/subset_of_gr3df97a.tif \ + tests/egm96_15_uncompressed_truncated.tif \ null \ generate_all_sql_in.cmake sql_filelist.cmake \ $(SQL_ORDERED_LIST) diff --git a/data/tests/egm96_15_uncompressed_truncated.tif b/data/tests/egm96_15_uncompressed_truncated.tif new file mode 100644 index 00000000..bd34a7e2 Binary files /dev/null and b/data/tests/egm96_15_uncompressed_truncated.tif differ diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 2dfcd6b6..cc692616 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -260,10 +260,7 @@ NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) { // --------------------------------------------------------------------------- -void NetworkChunkCache::clear() -{ - cache_.clear(); -} +void NetworkChunkCache::clear() { cache_.clear(); } // --------------------------------------------------------------------------- @@ -672,10 +669,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -void FileManager::clearCache() -{ - gNetworkChunkCache.clear(); -} +void FileManager::clearCache() { gNetworkChunkCache.clear(); } // --------------------------------------------------------------------------- diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 841d72b3..23d54663 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -147,3 +147,18 @@ target_link_libraries(gie_self_tests add_test(NAME gie_self_tests COMMAND gie_self_tests) set_property(TEST gie_self_tests PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + + +add_executable(test_network + main.cpp + test_network.cpp) +if(CURL_FOUND) + include_directories(${CURL_INCLUDE_DIR}) + target_link_libraries(test_network ${CURL_LIBRARY}) +endif() +target_link_libraries(test_network + GTest::gtest + ${PROJ_LIBRARIES}) +add_test(NAME test_network COMMAND test_network) +set_property(TEST test_network + PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index 57a03ca8..23ff5076 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -17,6 +17,7 @@ noinst_PROGRAMS += proj_context_test noinst_PROGRAMS += test_cpp_api noinst_PROGRAMS += gie_self_tests noinst_PROGRAMS += include_proj_h_from_c +noinst_PROGRAMS += test_network pj_transform_test_SOURCES = pj_transform_test.cpp main.cpp pj_transform_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ @@ -62,4 +63,11 @@ gie_self_tests-check: gie_self_tests include_proj_h_from_c_SOURCES = include_proj_h_from_c.c -check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check +test_network_SOURCES = test_network.cpp main.cpp +test_network_CXXFLAGS = @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ +test_network_LDADD = ../../src/libproj.la @GTEST_LIBS@ @CURL_LIBS@ + +test_network-check: test_network + PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network + +check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check test_network-check diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp new file mode 100644 index 00000000..6879e40d --- /dev/null +++ b/test/unit/test_network.cpp @@ -0,0 +1,497 @@ +/****************************************************************************** + * + * Project: PROJ + * Purpose: Test networking + * Author: Even Rouault + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "gtest_include.h" + +#include +#include + +#include "proj_internal.h" +#include + +#ifdef CURL_ENABLED +#include +#endif + +namespace { + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +static bool networkAccessOK = false; + +static size_t noop_curl_write_func(void *, size_t, size_t nmemb, void *) { + return nmemb; +} + +TEST(networking, initial_check) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return; + curl_easy_setopt(hCurlHandle, CURLOPT_URL, + "https://cdn.proj.org/ntf_r93.tif"); + + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, "0-1"); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, noop_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_cleanup(hCurlHandle); + + networkAccessOK = (response_code == 206); + if (!networkAccessOK) { + fprintf(stderr, "network access not working"); + } +} + +#endif + +// --------------------------------------------------------------------------- + +TEST(networking, basic) { + auto P = proj_create( + PJ_DEFAULT_CTX, + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg "); +#ifdef CURL_ENABLED + if (networkAccessOK) { + ASSERT_NE(P, nullptr); + } else { + ASSERT_EQ(P, nullptr); + return; + } + double lon = 2; + double lat = 49; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, nullptr, 0, 0, nullptr, 0, 0); + EXPECT_NEAR(lon, 1.9992776848, 1e-10); + EXPECT_NEAR(lat, 48.9999322600, 1e-10); + + proj_destroy(P); +#else + ASSERT_EQ(P, nullptr); +#endif +} + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +static void silent_logger(void *, int, const char *) {} + +TEST(networking, curl_invalid_resource) { + auto ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + auto P = proj_create( + ctx, "+proj=hgridshift +grids=https://i_do_not.exist/my.tif"); + proj_context_destroy(ctx); + ASSERT_EQ(P, nullptr); +} +#endif + +// --------------------------------------------------------------------------- + +struct Event { + virtual ~Event(); + std::string type{}; + PJ_CONTEXT *ctx = nullptr; +}; + +Event::~Event() = default; + +struct OpenEvent : public Event { + OpenEvent() { type = "OpenEvent"; } + + std::string url{}; + unsigned long long offset = 0; + size_t size_to_read = 0; + std::vector response{}; + int file_id = 0; +}; + +struct CloseEvent : public Event { + CloseEvent() { type = "CloseEvent"; } + + int file_id = 0; +}; + +struct GetHeaderValueEvent : public Event { + GetHeaderValueEvent() { type = "GetHeaderValueEvent"; } +}; + +struct GetFileSizeEvent : public Event { + GetFileSizeEvent() { type = "GetFileSizeEvent"; } +}; + +struct ReadRangeEvent : public Event { + ReadRangeEvent() { type = "ReadRangeEvent"; } + + unsigned long long offset = 0; + size_t size_to_read = 0; + std::vector response{}; + int file_id = 0; +}; + +struct GetLastErrorEvent : public Event { + GetLastErrorEvent() { type = "GetLastErrorEvent"; } +}; + +struct File {}; + +struct ExchangeWithCallback { + std::vector> events{}; + size_t nextEvent = 0; + bool error = false; + std::map mapIdToHandle{}; + + bool allConsumedAndNoError() const { + return nextEvent == events.size() && !error; + } +}; + +static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, void *user_data) { + auto exchange = static_cast(user_data); + if (exchange->error) + return nullptr; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to open(%s, %ld, %ld)\n", url, + (long)offset, (long)size_to_read); + exchange->error = true; + return nullptr; + } + auto openEvent = + dynamic_cast(exchange->events[exchange->nextEvent].get()); + if (!openEvent) { + fprintf(stderr, "unexpected call to open(%s, %ld, %ld). " + "Was expecting a %s event\n", + url, (long)offset, (long)size_to_read, + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return nullptr; + } + exchange->nextEvent++; + if (openEvent->ctx != ctx || openEvent->url != url || + openEvent->offset != offset || + openEvent->size_to_read != size_to_read) { + fprintf(stderr, "wrong call to open(%s, %ld, %ld). Was expecting " + "open(%s, %ld, %ld)\n", + url, (long)offset, (long)size_to_read, openEvent->url.c_str(), + (long)openEvent->offset, (long)openEvent->size_to_read); + exchange->error = true; + return nullptr; + } + memcpy(buffer, openEvent->response.data(), openEvent->response.size()); + *out_size_read = openEvent->response.size(); + + auto handle = reinterpret_cast(new File()); + exchange->mapIdToHandle[openEvent->file_id] = handle; + return handle; +} + +static void close_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, + void *user_data) { + auto exchange = static_cast(user_data); + if (exchange->error) + return; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to close()\n"); + exchange->error = true; + return; + } + auto closeEvent = + dynamic_cast(exchange->events[exchange->nextEvent].get()); + if (!closeEvent) { + fprintf(stderr, "unexpected call to close(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return; + } + if (closeEvent->ctx != ctx) { + fprintf(stderr, "close() called with bad context\n"); + exchange->error = true; + return; + } + if (exchange->mapIdToHandle[closeEvent->file_id] != handle) { + fprintf(stderr, "close() called with bad handle\n"); + exchange->error = true; + return; + } + exchange->nextEvent++; + delete reinterpret_cast(handle); +} + +static const char *get_header_value_cbk(PJ_CONTEXT * /* ctx */, + PROJ_NETWORK_HANDLE * /*handle*/, + const char * /*header_name*/, + void *user_data) { + auto exchange = static_cast(user_data); + if (exchange->error) + return nullptr; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to get_header_value()\n"); + exchange->error = true; + return nullptr; + } + auto getHeaderValueEvent = dynamic_cast( + exchange->events[exchange->nextEvent].get()); + if (!getHeaderValueEvent) { + fprintf(stderr, "unexpected call to get_header_value(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return nullptr; + } + exchange->nextEvent++; + return nullptr; +} + +static unsigned long long get_file_size_cbk(PJ_CONTEXT * /* ctx */, + PROJ_NETWORK_HANDLE * /*handle*/, + void *user_data) { + auto exchange = static_cast(user_data); + if (exchange->error) + return 0; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to get_file_size()\n"); + exchange->error = true; + return 0; + } + auto getFileSizeEvent = dynamic_cast( + exchange->events[exchange->nextEvent].get()); + if (!getFileSizeEvent) { + fprintf(stderr, "unexpected call to get_file_size(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return 0; + } + exchange->nextEvent++; + return 0; +} + +static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, + unsigned long long offset, size_t size_to_read, + void *buffer, void *user_data) { + auto exchange = static_cast(user_data); + if (exchange->error) + return 0; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to read_range(%ld, %ld)\n", + (long)offset, (long)size_to_read); + exchange->error = true; + return 0; + } + auto readRangeEvent = dynamic_cast( + exchange->events[exchange->nextEvent].get()); + if (!readRangeEvent) { + fprintf(stderr, "unexpected call to read_range(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return 0; + } + if (exchange->mapIdToHandle[readRangeEvent->file_id] != handle) { + fprintf(stderr, "read_range() called with bad handle\n"); + exchange->error = true; + return 0; + } + if (readRangeEvent->ctx != ctx || readRangeEvent->offset != offset || + readRangeEvent->size_to_read != size_to_read) { + fprintf(stderr, "wrong call to read_range(%ld, %ld). Was expecting " + "read_range(%ld, %ld)\n", + (long)offset, (long)size_to_read, (long)readRangeEvent->offset, + (long)readRangeEvent->size_to_read); + exchange->error = true; + return 0; + } + memcpy(buffer, readRangeEvent->response.data(), + readRangeEvent->response.size()); + exchange->nextEvent++; + return readRangeEvent->response.size(); +} + +static const char *get_last_error_cbk(PJ_CONTEXT * /*ctx*/, + PROJ_NETWORK_HANDLE *, void *user_data) { + auto exchange = static_cast(user_data); + if (exchange->error) + return ""; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to get_last_error()\n"); + exchange->error = true; + return ""; + } + auto getLastErrorEvent = dynamic_cast( + exchange->events[exchange->nextEvent].get()); + if (!getLastErrorEvent) { + fprintf( + stderr, + "unexpected call to get_last_error(). Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return ""; + } + exchange->nextEvent++; + return ""; +} + +TEST(networking, custom) { + auto ctx = proj_context_create(); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks( + ctx, open_cbk, close_cbk, get_header_value_cbk, get_file_size_cbk, + read_range_cbk, get_last_error_cbk, &exchange)); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/my.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/my.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new ReadRangeEvent()); + event->ctx = ctx; + event->offset = 3670016; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 2.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = -49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 2.25); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + // Once again ! No network access + + P = proj_create(ctx, + "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1"); + ASSERT_NE(P, nullptr); + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +} // namespace -- cgit v1.2.3 From 9d0bd793b552e248a10f9ff8b6c62d942fe437a1 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Dec 2019 15:58:47 +0100 Subject: Network: remove dedicated get_file_size() callback to use get_header_value(), and test --- data/Makefile.am | 1 + data/tests/test_vgrid_single_strip_truncated.tif | Bin 0 -> 550 bytes src/filemanager.cpp | 93 ++++++++++++----- src/proj.h | 10 +- src/proj_internal.h | 1 - test/unit/test_network.cpp | 123 +++++++++++++++++------ 6 files changed, 162 insertions(+), 66 deletions(-) create mode 100644 data/tests/test_vgrid_single_strip_truncated.tif diff --git a/data/Makefile.am b/data/Makefile.am index 59ab4e83..3778a2be 100644 --- a/data/Makefile.am +++ b/data/Makefile.am @@ -81,6 +81,7 @@ EXTRA_DIST = GL27 nad.lst nad27 nad83 \ tests/test_hgrid_with_subgrid_no_grid_name.tif \ tests/subset_of_gr3df97a.tif \ tests/egm96_15_uncompressed_truncated.tif \ + tests/test_vgrid_single_strip_truncated.tif \ null \ generate_all_sql_in.cmake sql_filelist.cmake \ $(SQL_ORDERED_LIST) diff --git a/data/tests/test_vgrid_single_strip_truncated.tif b/data/tests/test_vgrid_single_strip_truncated.tif new file mode 100644 index 00000000..9a0030f6 Binary files /dev/null and b/data/tests/test_vgrid_single_strip_truncated.tif differ diff --git a/src/filemanager.cpp b/src/filemanager.cpp index cc692616..ea0a63ea 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -266,6 +266,13 @@ void NetworkChunkCache::clear() { cache_.clear(); } static NetworkChunkCache gNetworkChunkCache{}; +struct FileProperties { + unsigned long long size; +}; + +static lru11::Cache + gNetworkFileProperties{}; + // --------------------------------------------------------------------------- class NetworkFile : public File { @@ -275,6 +282,7 @@ class NetworkFile : public File { unsigned long long m_pos = 0; size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; + unsigned long long m_filesize; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; @@ -282,9 +290,10 @@ class NetworkFile : public File { protected: NetworkFile(PJ_CONTEXT *ctx, const std::string &url, PROJ_NETWORK_HANDLE *handle, - unsigned long long lastDownloadOffset) + unsigned long long lastDownloadOffset, + unsigned long long filesize) : m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset) {} + m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} public: ~NetworkFile() override; @@ -300,9 +309,14 @@ class NetworkFile : public File { std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { if (gNetworkChunkCache.get(filename, 0)) { - return std::unique_ptr( - new NetworkFile(ctx, filename, nullptr, - std::numeric_limits::max())); + unsigned long long filesize = 0; + FileProperties props; + if (gNetworkFileProperties.tryGet(filename, props)) { + filesize = props.size; + } + return std::unique_ptr(new NetworkFile( + ctx, filename, nullptr, + std::numeric_limits::max(), filesize)); } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; @@ -311,8 +325,24 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { &size_read, ctx->networking.user_data); buffer.resize(size_read); gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + + unsigned long long filesize = 0; + if (handle) { + const char *contentRange = ctx->networking.get_header_value( + ctx, handle, "Content-Range", ctx->networking.user_data); + if (contentRange) { + const char *slash = strchr(contentRange, '/'); + if (slash) { + filesize = std::stoull(slash + 1); + FileProperties props; + props.size = filesize; + gNetworkFileProperties.insert(filename, props); + } + } + } + return std::unique_ptr( - handle ? new NetworkFile(ctx, filename, handle, size_read) + handle ? new NetworkFile(ctx, filename, handle, size_read, filesize) : nullptr); } } @@ -431,21 +461,7 @@ bool NetworkFile::seek(unsigned long long offset, int whence) { } else { if (offset != 0) return false; - if (!m_handle) { - size_t nRead = 0; - char dummy; - m_handle = - m_ctx->networking.open(m_ctx, m_url.c_str(), 0, 1, &dummy, - &nRead, m_ctx->networking.user_data); - if (!m_handle) { - return false; - } - } - const auto filesize = m_ctx->networking.get_file_size( - m_ctx, m_handle, m_ctx->networking.user_data); - if (filesize == 0) - return false; - m_pos = filesize; + m_pos = m_filesize; } return true; } @@ -484,6 +500,7 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { struct CurlFileHandle { CURL *m_handle = nullptr; std::string m_headers; + std::string m_lastval{}; CurlFileHandle(const CurlFileHandle &) = delete; CurlFileHandle &operator=(const CurlFileHandle &) = delete; @@ -623,6 +640,27 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, return std::min(size_to_read, body.size()); } +// --------------------------------------------------------------------------- + +static const char *pj_curl_get_header_value(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *raw_handle, + const char *header_name, void *) { + auto handle = reinterpret_cast(raw_handle); + auto pos = ci_find(handle->m_headers, header_name); + if (pos == std::string::npos) + return nullptr; + const char *c_str = handle->m_headers.c_str(); + if (c_str[pos] == ':') + pos++; + while (c_str[pos] == ' ') + pos++; + auto posEnd = pos; + while (c_str[posEnd] != '\n' && c_str[posEnd] != '\0') + posEnd++; + handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos); + return handle->m_lastval.c_str(); +} + #else // --------------------------------------------------------------------------- @@ -660,6 +698,7 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { ctx->networking.open = pj_curl_open; ctx->networking.close = pj_curl_close; ctx->networking.read_range = pj_curl_read_range; + ctx->networking.get_header_value = pj_curl_get_header_value; #else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; @@ -669,7 +708,10 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { // --------------------------------------------------------------------------- -void FileManager::clearCache() { gNetworkChunkCache.clear(); } +void FileManager::clearCache() { + gNetworkChunkCache.clear(); + gNetworkFileProperties.clear(); +} // --------------------------------------------------------------------------- @@ -687,7 +729,6 @@ NS_PROJ_END * @param open_cbk Callback to open a remote file given its URL * @param close_cbk Callback to close a remote file. * @param get_header_value_cbk Callback to get HTTP headers - * @param get_file_size_cbk Callback to get the size of the remote file. * @param read_range_cbk Callback to read a range of bytes inside a remote file. * @param get_last_error_cbk Callback to get last error message. * @param user_data Arbitrary pointer provided by the user, and passed to the @@ -698,20 +739,18 @@ int proj_context_set_network_callbacks( PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_get_file_size_cbk_type get_file_size_cbk, proj_network_read_range_type read_range_cbk, proj_network_get_last_error_type get_last_error_cbk, void *user_data) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } - if (!open_cbk || !close_cbk || !get_header_value_cbk || - !get_file_size_cbk || !read_range_cbk || !get_last_error_cbk) { + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk || + !get_last_error_cbk) { return false; } ctx->networking.open = open_cbk; ctx->networking.close = close_cbk; ctx->networking.get_header_value = get_header_value_cbk; - ctx->networking.get_file_size = get_file_size_cbk; ctx->networking.read_range = read_range_cbk; ctx->networking.get_last_error = get_last_error_cbk; ctx->networking.user_data = user_data; diff --git a/src/proj.h b/src/proj.h index 5f080285..95bbacd1 100644 --- a/src/proj.h +++ b/src/proj.h @@ -367,8 +367,7 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; * amount of bytes read (== size_to_read if the file is larger than size_to_read). * During this read, the implementation should make sure to store the HTTP * headers from the server response to be able to respond to - * proj_network_get_header_value_cbk_type and proj_network_get_file_size_cbk_type - * callbacks. + * proj_network_get_header_value_cbk_type callback. * * @return a non-NULL opaque handle in case of success. */ @@ -393,12 +392,6 @@ typedef const char* (*proj_network_get_header_value_cbk_type)( const char* header_name, void* user_data); -/** Network access: get file size */ -typedef unsigned long long (*proj_network_get_file_size_cbk_type)( - PJ_CONTEXT* ctx, - PROJ_NETWORK_HANDLE* handle, - void* user_data); - /** Network access: read range * * Read size_to_read bytes from handle, starting at offset, into @@ -424,7 +417,6 @@ int PROJ_DLL proj_context_set_network_callbacks( proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_get_file_size_cbk_type get_file_size_cbk, proj_network_read_range_type read_range_cbk, proj_network_get_last_error_type get_last_error_cbk, void* user_data); diff --git a/src/proj_internal.h b/src/proj_internal.h index aefeea39..0c148d02 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -671,7 +671,6 @@ struct projNetworkCallbacksAndData proj_network_open_cbk_type open = nullptr; proj_network_close_cbk_type close = nullptr; proj_network_get_header_value_cbk_type get_header_value = nullptr; - proj_network_get_file_size_cbk_type get_file_size = nullptr; proj_network_read_range_type read_range = nullptr; proj_network_get_last_error_type get_last_error = nullptr; void* user_data = nullptr; diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index 6879e40d..5940814b 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -148,10 +148,10 @@ struct CloseEvent : public Event { struct GetHeaderValueEvent : public Event { GetHeaderValueEvent() { type = "GetHeaderValueEvent"; } -}; -struct GetFileSizeEvent : public Event { - GetFileSizeEvent() { type = "GetFileSizeEvent"; } + int file_id = 0; + std::string key{}; + std::string value{}; }; struct ReadRangeEvent : public Event { @@ -255,9 +255,9 @@ static void close_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, delete reinterpret_cast(handle); } -static const char *get_header_value_cbk(PJ_CONTEXT * /* ctx */, - PROJ_NETWORK_HANDLE * /*handle*/, - const char * /*header_name*/, +static const char *get_header_value_cbk(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + const char *header_name, void *user_data) { auto exchange = static_cast(user_data); if (exchange->error) @@ -276,32 +276,25 @@ static const char *get_header_value_cbk(PJ_CONTEXT * /* ctx */, exchange->error = true; return nullptr; } - exchange->nextEvent++; - return nullptr; -} - -static unsigned long long get_file_size_cbk(PJ_CONTEXT * /* ctx */, - PROJ_NETWORK_HANDLE * /*handle*/, - void *user_data) { - auto exchange = static_cast(user_data); - if (exchange->error) - return 0; - if (exchange->nextEvent >= exchange->events.size()) { - fprintf(stderr, "unexpected call to get_file_size()\n"); + if (getHeaderValueEvent->ctx != ctx) { + fprintf(stderr, "get_header_value() called with bad context\n"); exchange->error = true; - return 0; + return nullptr; } - auto getFileSizeEvent = dynamic_cast( - exchange->events[exchange->nextEvent].get()); - if (!getFileSizeEvent) { - fprintf(stderr, "unexpected call to get_file_size(). " - "Was expecting a %s event\n", - exchange->events[exchange->nextEvent]->type.c_str()); + if (getHeaderValueEvent->key != header_name) { + fprintf(stderr, "wrong call to get_header_value(%s). Was expecting " + "get_header_value(%s)\n", + header_name, getHeaderValueEvent->key.c_str()); exchange->error = true; - return 0; + return nullptr; + } + if (exchange->mapIdToHandle[getHeaderValueEvent->file_id] != handle) { + fprintf(stderr, "get_header_value() called with bad handle\n"); + exchange->error = true; + return nullptr; } exchange->nextEvent++; - return 0; + return getHeaderValueEvent->value.c_str(); } static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, @@ -373,8 +366,8 @@ TEST(networking, custom) { auto ctx = proj_context_create(); ExchangeWithCallback exchange; ASSERT_TRUE(proj_context_set_network_callbacks( - ctx, open_cbk, close_cbk, get_header_value_cbk, get_file_size_cbk, - read_range_cbk, get_last_error_cbk, &exchange)); + ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, + get_last_error_cbk, &exchange)); { std::unique_ptr event(new OpenEvent()); @@ -395,6 +388,14 @@ TEST(networking, custom) { fclose(f); exchange.events.emplace_back(std::move(event)); } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "dummy"; // dummy value: not used + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } { std::unique_ptr event(new CloseEvent()); event->ctx = ctx; @@ -494,4 +495,68 @@ TEST(networking, custom) { proj_context_destroy(ctx); } +// --------------------------------------------------------------------------- + +TEST(networking, getfilesize) { + auto ctx = proj_context_create(); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks( + ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, + get_last_error_cbk, &exchange)); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/getfilesize.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/test_vgrid_single_strip_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 550, f), 550U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes 0-16383/4153510"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_destroy(P); + + P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_destroy(P); + + proj_context_destroy(ctx); +} + } // namespace -- cgit v1.2.3 From 0a1f1fe469029ae31591dc8b51d20f5617496128 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 23 Dec 2019 23:12:26 +0100 Subject: Network: only enable it if PROJ_NETWORK=ON or proj_context_set_enable_network(ctx, true) --- scripts/reference_exported_symbols.txt | 1 + src/filemanager.cpp | 58 ++++++++++++++++++++++++++++++++-- src/proj.h | 3 ++ src/proj_internal.h | 4 +++ test/unit/test_network.cpp | 49 +++++++++++++++++++++++++--- 5 files changed, 108 insertions(+), 7 deletions(-) diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index 6e4e5302..9d7d400a 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -794,6 +794,7 @@ proj_context_get_use_proj4_init_rules proj_context_guess_wkt_dialect proj_context_set_autoclose_database proj_context_set_database_path +proj_context_set_enable_network proj_context_set_file_finder proj_context_set_network_callbacks proj_context_set(PJconsts*, projCtx_t*) diff --git a/src/filemanager.cpp b/src/filemanager.cpp index ea0a63ea..cd738d5e 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -63,9 +63,9 @@ class MyMutex { //! @cond Doxygen_Suppress -NS_PROJ_START +using namespace NS_PROJ::internal; -using namespace internal; +NS_PROJ_START // --------------------------------------------------------------------------- @@ -488,6 +488,14 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { } #endif if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + if (!pj_context_is_network_enabled(ctx)) { + pj_log( + ctx, PJ_LOG_ERROR, + "Attempt at accessing remote resource not authorized. Either " + "set PROJ_NETWORK=ON or " + "proj_context_set_enable_network(ctx, TRUE)"); + return nullptr; + } return NetworkFile::open(ctx, filename); } return FileStdio::open(ctx, filename); @@ -756,3 +764,49 @@ int proj_context_set_network_callbacks( ctx->networking.user_data = user_data; return true; } + +// --------------------------------------------------------------------------- + +/** Enable or disable network access. +* +* @param ctx PROJ context, or NULL +* @param enable TRUE if network access is allowed. +* @return TRUE if network access is possible. That is either libcurl is +* available, or an alternate interface has been set. +*/ +int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + ctx->networking.enabled_env_variable_checked = true; + ctx->networking.enabled = enable != FALSE; +#ifdef CURL_ENABLED + return ctx->networking.enabled; +#else + return ctx->networking.enabled && + ctx->networking.open != NS_PROJ::no_op_network_open; +#endif +} + +// --------------------------------------------------------------------------- + +//! @cond Doxygen_Suppress + +bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (ctx->networking.enabled_env_variable_checked) { + return ctx->networking.enabled; + } + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled && enabled[0] != '\0') { + ctx->networking.enabled = ci_equal(enabled, "ON") || + ci_equal(enabled, "YES") || + ci_equal(enabled, "TRUE"); + } + ctx->networking.enabled_env_variable_checked = true; + return ctx->networking.enabled; +} + +//! @endcond diff --git a/src/proj.h b/src/proj.h index 95bbacd1..4f7878a4 100644 --- a/src/proj.h +++ b/src/proj.h @@ -421,6 +421,9 @@ int PROJ_DLL proj_context_set_network_callbacks( proj_network_get_last_error_type get_last_error_cbk, void* user_data); +int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, + int enabled); + /*! @cond Doxygen_Suppress */ /* Manage the transformation definition object PJ */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 0c148d02..d54d8fb9 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -668,6 +668,8 @@ struct projCppContext; struct projNetworkCallbacksAndData { + bool enabled = false; + bool enabled_env_variable_checked = false; // whereas we have checked PROJ_NETWORK env variable proj_network_open_cbk_type open = nullptr; proj_network_close_cbk_type close = nullptr; proj_network_get_header_value_cbk_type get_header_value = nullptr; @@ -824,6 +826,8 @@ PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); +bool pj_context_is_network_enabled(PJ_CONTEXT* ctx); + /* classic public API */ #include "proj_api.h" diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index 5940814b..ba592da4 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -30,6 +30,7 @@ #include #include +#include #include "proj_internal.h" #include @@ -77,18 +78,54 @@ TEST(networking, initial_check) { // --------------------------------------------------------------------------- +static void silent_logger(void *, int, const char *) {} + +// --------------------------------------------------------------------------- + TEST(networking, basic) { - auto P = proj_create( - PJ_DEFAULT_CTX, + const char *pipeline = "+proj=pipeline " "+step +proj=unitconvert +xy_in=deg +xy_out=rad " "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " - "+step +proj=unitconvert +xy_in=rad +xy_out=deg "); + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + // network access disabled by default + auto ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + auto P = proj_create(ctx, pipeline); + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + +#ifdef CURL_ENABLED + // enable through env variable + ctx = proj_context_create(); + putenv(const_cast("PROJ_NETWORK=ON")); + P = proj_create(ctx, pipeline); + if (networkAccessOK) { + ASSERT_NE(P, nullptr); + } + proj_destroy(P); + proj_context_destroy(ctx); + putenv(const_cast("PROJ_NETWORK=")); +#endif + + // still disabled + ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + P = proj_create(ctx, pipeline); + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + + // enable through API + ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + P = proj_create(ctx, pipeline); #ifdef CURL_ENABLED if (networkAccessOK) { ASSERT_NE(P, nullptr); } else { ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); return; } double lon = 2; @@ -102,16 +139,16 @@ TEST(networking, basic) { #else ASSERT_EQ(P, nullptr); #endif + proj_context_destroy(ctx); } // --------------------------------------------------------------------------- #ifdef CURL_ENABLED -static void silent_logger(void *, int, const char *) {} - TEST(networking, curl_invalid_resource) { auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); proj_log_func(ctx, nullptr, silent_logger); auto P = proj_create( ctx, "+proj=hgridshift +grids=https://i_do_not.exist/my.tif"); @@ -364,6 +401,7 @@ static const char *get_last_error_cbk(PJ_CONTEXT * /*ctx*/, TEST(networking, custom) { auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); ExchangeWithCallback exchange; ASSERT_TRUE(proj_context_set_network_callbacks( ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, @@ -499,6 +537,7 @@ TEST(networking, custom) { TEST(networking, getfilesize) { auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); ExchangeWithCallback exchange; ASSERT_TRUE(proj_context_set_network_callbacks( ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, -- cgit v1.2.3 From c4589fbe42e5fea07a03919d3484164f5fb70dd3 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 18:44:45 +0100 Subject: Network: automatically use CDN resources when local resources not available, and networking enabled --- docs/source/apps/cs2cs.rst | 8 ++ docs/source/apps/projinfo.rst | 10 +- docs/source/resource_files.rst | 5 + docs/source/usage/environmentvars.rst | 9 ++ include/proj/coordinateoperation.hpp | 18 ++- .../proj/internal/coordinateoperation_internal.hpp | 15 ++- include/proj/io.hpp | 8 +- scripts/reference_exported_symbols.txt | 13 +- src/4D_api.cpp | 7 +- src/apps/projinfo.cpp | 30 +++-- src/filemanager.cpp | 15 ++- src/filemanager.hpp | 5 +- src/grids.cpp | 26 ++-- src/iso19111/c_api.cpp | 20 +++- src/iso19111/coordinateoperation.cpp | 131 +++++++++++++++------ src/iso19111/factory.cpp | 77 +++++++----- src/open_lib.cpp | 59 +++++++--- src/proj.h | 6 + src/proj_internal.h | 4 +- src/transformations/hgridshift.cpp | 40 +++++-- src/transformations/vgridshift.cpp | 65 ++++++++-- test/unit/Makefile.am | 2 +- test/unit/test_factory.cpp | 58 ++++----- test/unit/test_network.cpp | 86 ++++++++++++++ test/unit/test_operation.cpp | 18 +-- 25 files changed, 547 insertions(+), 188 deletions(-) diff --git a/docs/source/apps/cs2cs.rst b/docs/source/apps/cs2cs.rst index d7f0e9ad..a4069472 100644 --- a/docs/source/apps/cs2cs.rst +++ b/docs/source/apps/cs2cs.rst @@ -175,6 +175,14 @@ normally be in DMS format (use ``-f %.12f`` for decimal degrees with 12 decimal places), while projected (cartesian) coordinates will be in linear (meter, feet) units. +Use of remote grids +------------------- + +.. versionadded:: 7.0.0 + +If the :envvar:`PROJ_NETWORK` environment variable is set to ``ON``, +:program:`cs2cs` will attempt to use remote grids stored on CDN (Content +Delivery Network) storage, when they are not available locally. Examples ******** diff --git a/docs/source/apps/projinfo.rst b/docs/source/apps/projinfo.rst index 6236056d..a9b6aecc 100644 --- a/docs/source/apps/projinfo.rst +++ b/docs/source/apps/projinfo.rst @@ -20,7 +20,7 @@ Synopsis | [[--area name_or_code] | [--bbox west_long,south_lat,east_long,north_lat]] | [--spatial-test contains|intersects] | [--crs-extent-use none|both|intersection|smallest] - | [--grid-check none|discard_missing|sort] [--show-superseded] + | [--grid-check none|discard_missing|sort|known_available] [--show-superseded] | [--pivot-crs always|if_no_direct_transformation|never|{auth:code[,auth:code]*}] | [--boundcrs-to-wgs84] | [--main-db-path path] [--aux-db-path path]* @@ -148,12 +148,13 @@ The following control parameters can appear in any order: .. note:: only used for coordinate operation computation -.. option:: --grid-check none|discard_missing|sort +.. option:: --grid-check none|discard_missing|sort|known_available Specify how the presence or absence of a horizontal or vertical shift grid required for a coordinate operation affects the results returned when researching coordinate operations between 2 CRS. - The default strategy is ``sort``: in that case, all candidate + The default strategy is ``sort`` (if :envvar:`PROJ_NETWORK` is not defined). + In that case, all candidate operations are returned, but the actual availability of the grids is used to determine the sorting order. That is, if a coordinate operation involves using a grid that is not available in the PROJ resource directories @@ -163,6 +164,9 @@ The following control parameters can appear in any order: this returns the results as if all the grids where available. The ``discard_missing`` strategy discards results that involve grids not present in the PROJ resource directories. + The ``known_available`` strategy discards results that involve grids not + present in the PROJ resource directories and that are not known of the CDN. + This is the default strategy is :envvar:`PROJ_NETWORK` is set to ``ON``. .. note:: only used for coordinate operation computation diff --git a/docs/source/resource_files.rst b/docs/source/resource_files.rst index 13b9386d..69cf2f95 100644 --- a/docs/source/resource_files.rst +++ b/docs/source/resource_files.rst @@ -40,6 +40,11 @@ The following paths are checked in order: PROJ installation is not moved somewhere else. - The current directory +When networking capabilities are enabled, either by API with the +:c:func:`proj_context_set_enable_network` function or when the +:envvar:`PROJ_NETWORK` environment variable is set to ``ON``, PROJ will +attempt to use remote grids stored on CDN (Content Delivery Network) storage. + .. _proj-db: proj.db diff --git a/docs/source/usage/environmentvars.rst b/docs/source/usage/environmentvars.rst index 457432a0..00058716 100644 --- a/docs/source/usage/environmentvars.rst +++ b/docs/source/usage/environmentvars.rst @@ -56,3 +56,12 @@ done by setting the variable with no content:: Set the debug level of PROJ. The default debug level is zero, which results in no debug output when using PROJ. A number from 1-3, whit 3 being the most verbose setting. + +.. envvar:: PROJ_NETWORK + + .. versionadded:: 7.0.0 + + If set to ON, enable the capability to use remote grids stored on CDN + (Content Delivery Network) storage, when grids are not available locally. + Alternatively, the :c:func:`proj_context_set_enable_network` function can + be used. diff --git a/include/proj/coordinateoperation.hpp b/include/proj/coordinateoperation.hpp index 55bc28a0..bee3eff2 100644 --- a/include/proj/coordinateoperation.hpp +++ b/include/proj/coordinateoperation.hpp @@ -144,10 +144,12 @@ class PROJ_GCC_DLL CoordinateOperation : public common::ObjectUsage, /** \brief Return grids needed by an operation. */ PROJ_DLL virtual std::set - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const = 0; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const = 0; PROJ_DLL bool - isPROJInstantiable(const io::DatabaseContextPtr &databaseContext) const; + isPROJInstantiable(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const; PROJ_DLL bool hasBallparkTransformation() const; @@ -595,7 +597,8 @@ class PROJ_GCC_DLL SingleOperation : virtual public CoordinateOperation { std::vector()); PROJ_DLL std::set - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; PROJ_DLL std::list validateParameters() const; @@ -1672,7 +1675,8 @@ class PROJ_GCC_DLL ConcatenatedOperation final : public CoordinateOperation { bool checkExtent); // throw InvalidOperation PROJ_DLL std::set - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; PROJ_PRIVATE : @@ -1801,6 +1805,12 @@ class PROJ_GCC_DLL CoordinateOperationContext { /** Ignore grid availability at all. Results will be presented as if * all grids were available. */ IGNORE_GRID_AVAILABILITY, + + /** Results will be presented as if grids known to PROJ (that is + * registered in the grid_alternatives table of its database) were + * available. Used typically when networking is enabled. + */ + KNOWN_AVAILABLE, }; PROJ_DLL void setGridAvailabilityUse(GridAvailabilityUse use); diff --git a/include/proj/internal/coordinateoperation_internal.hpp b/include/proj/internal/coordinateoperation_internal.hpp index 7ae2cd78..921021ec 100644 --- a/include/proj/internal/coordinateoperation_internal.hpp +++ b/include/proj/internal/coordinateoperation_internal.hpp @@ -177,8 +177,10 @@ class InverseConversion : public Conversion, public InverseCoordinateOperation { // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded' // via dominance std::set - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override { - return SingleOperation::gridsNeeded(databaseContext); + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override { + return SingleOperation::gridsNeeded(databaseContext, + considerKnownGridsAsAvailable); } #endif @@ -227,8 +229,10 @@ class InverseTransformation : public Transformation, // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded' // via dominance std::set - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override { - return SingleOperation::gridsNeeded(databaseContext); + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override { + return SingleOperation::gridsNeeded(databaseContext, + considerKnownGridsAsAvailable); } #endif @@ -269,7 +273,8 @@ class PROJBasedOperation : public SingleOperation { bool hasRoughTransformation); std::set - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; protected: PROJBasedOperation(const PROJBasedOperation &) = default; diff --git a/include/proj/io.hpp b/include/proj/io.hpp index be293eb9..2839896e 100644 --- a/include/proj/io.hpp +++ b/include/proj/io.hpp @@ -832,6 +832,7 @@ class PROJ_GCC_DLL DatabaseContext { bool &inverse) const; PROJ_DLL bool lookForGridInfo(const std::string &projFilename, + bool considerKnownGridsAsAvailable, std::string &fullFilename, std::string &packageName, std::string &url, bool &directDownload, bool &openLicense, @@ -1050,7 +1051,8 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, bool tryReverseOrder = false, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + bool tryReverseOrder = false, bool reportOnlyIntersectingTransformations = false, const metadata::ExtentPtr &intersectingExtent1 = nullptr, const metadata::ExtentPtr &intersectingExtent2 = nullptr) const; @@ -1060,7 +1062,7 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector> &intermediateCRSAuthCodes, ObjectType allowedIntermediateObjectType = ObjectType::CRS, @@ -1126,7 +1128,7 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector &allowedAuthorities, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const; diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index 9d7d400a..d2724825 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -299,8 +299,8 @@ osgeo::proj::io::AuthorityFactory::create(dropbox::oxygen::nn const&, std::shared_ptr const&) const -osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, std::vector, std::allocator > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector > const&, std::shared_ptr const&, std::shared_ptr const&) const +osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, bool, bool, std::shared_ptr const&, std::shared_ptr const&) const +osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, std::vector, std::allocator > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector > const&, std::shared_ptr const&, std::shared_ptr const&) const osgeo::proj::io::AuthorityFactory::createGeodeticCRS(std::string const&) const osgeo::proj::io::AuthorityFactory::createGeodeticDatum(std::string const&) const osgeo::proj::io::AuthorityFactory::createGeographicCRS(std::string const&) const @@ -330,7 +330,7 @@ osgeo::proj::io::DatabaseContext::getDatabaseStructure() const osgeo::proj::io::DatabaseContext::getMetadata(char const*) const osgeo::proj::io::DatabaseContext::getPath() const osgeo::proj::io::DatabaseContext::getSqliteHandle() const -osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, std::string&, std::string&, std::string&, bool&, bool&, bool&) const +osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, bool, std::string&, std::string&, std::string&, bool&, bool&, bool&) const osgeo::proj::io::FactoryException::~FactoryException() osgeo::proj::io::FactoryException::FactoryException(char const*) osgeo::proj::io::FactoryException::FactoryException(osgeo::proj::io::FactoryException const&) @@ -465,7 +465,7 @@ osgeo::proj::metadata::VerticalExtent::~VerticalExtent() osgeo::proj::operation::ConcatenatedOperation::~ConcatenatedOperation() osgeo::proj::operation::ConcatenatedOperation::createComputeMetadata(std::vector >, std::allocator > > > const&, bool) osgeo::proj::operation::ConcatenatedOperation::create(osgeo::proj::util::PropertyMap const&, std::vector >, std::allocator > > > const&, std::vector >, std::allocator > > > const&) -osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr const&) const +osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr const&, bool) const osgeo::proj::operation::ConcatenatedOperation::inverse() const osgeo::proj::operation::ConcatenatedOperation::operations() const osgeo::proj::operation::Conversion::~Conversion() @@ -575,7 +575,7 @@ osgeo::proj::operation::CoordinateOperationFactory::createOperation(dropbox::oxy osgeo::proj::operation::CoordinateOperationFactory::createOperations(dropbox::oxygen::nn > const&, dropbox::oxygen::nn > const&, dropbox::oxygen::nn > > const&) const osgeo::proj::operation::CoordinateOperation::hasBallparkTransformation() const osgeo::proj::operation::CoordinateOperation::interpolationCRS() const -osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr const&) const +osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr const&, bool) const osgeo::proj::operation::CoordinateOperation::normalizeForVisualization() const osgeo::proj::operation::CoordinateOperation::operationVersion() const osgeo::proj::operation::CoordinateOperation::shallowClone() const @@ -621,7 +621,7 @@ osgeo::proj::operation::ParameterValue::value() const osgeo::proj::operation::ParameterValue::valueFile() const osgeo::proj::operation::PointMotionOperation::~PointMotionOperation() osgeo::proj::operation::SingleOperation::createPROJBased(osgeo::proj::util::PropertyMap const&, std::string const&, std::shared_ptr const&, std::shared_ptr const&, std::vector >, std::allocator > > > const&) -osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr const&) const +osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr const&, bool) const osgeo::proj::operation::SingleOperation::method() const osgeo::proj::operation::SingleOperation::parameterValue(int) const osgeo::proj::operation::SingleOperation::parameterValueMeasure(int) const @@ -707,6 +707,7 @@ pj_chomp(char*) pj_cleanup_lock pj_clear_initcache pj_compare_datums +pj_context_is_network_enabled(projCtx_t*) pj_ctx_alloc pj_ctx_fclose pj_ctx_fgets diff --git a/src/4D_api.cpp b/src/4D_api.cpp index efb4a86a..cee8262e 100644 --- a/src/4D_api.cpp +++ b/src/4D_api.cpp @@ -262,7 +262,7 @@ similarly, but prefers the 2D resp. 3D interfaces if available. auto coordOperation = dynamic_cast< NS_PROJ::operation::CoordinateOperation*>(alt.pj->iso_obj.get()); if( coordOperation ) { - if( coordOperation->gridsNeeded(dbContext).empty() ) { + if( coordOperation->gridsNeeded(dbContext, true).empty() ) { if( P->iCurCoordOp != i ) { std::string msg("Using coordinate operation "); msg += alt.name; @@ -1117,7 +1117,10 @@ PJ *proj_create_crs_to_crs_from_pj (PJ_CONTEXT *ctx, const PJ *source_crs, cons proj_operation_factory_context_set_spatial_criterion( ctx, operation_ctx, PROJ_SPATIAL_CRITERION_PARTIAL_INTERSECTION); proj_operation_factory_context_set_grid_availability_use( - ctx, operation_ctx, PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); + ctx, operation_ctx, + pj_context_is_network_enabled(ctx) ? + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); auto op_list = proj_create_operations(ctx, source_crs, target_crs, operation_ctx); diff --git a/src/apps/projinfo.cpp b/src/apps/projinfo.cpp index fd9b2f46..f13e526b 100644 --- a/src/apps/projinfo.cpp +++ b/src/apps/projinfo.cpp @@ -87,7 +87,8 @@ static void usage() { << " [--spatial-test contains|intersects]" << std::endl << " [--crs-extent-use none|both|intersection|smallest]" << std::endl - << " [--grid-check none|discard_missing|sort] " + << " [--grid-check " + "none|discard_missing|sort|known_available] " "[--show-superseded]" << std::endl << " [--pivot-crs always|if_no_direct_transformation|" @@ -509,7 +510,7 @@ static void outputObject( auto op = dynamic_cast(obj.get()); if (op && dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); bool firstWarning = true; for (const auto &grid : setGrids) { if (!grid.available) { @@ -525,6 +526,7 @@ static void outputObject( if (!grid.url.empty()) { std::cout << " at " << grid.url; } + std::cout << ", or on CDN"; } else if (!grid.url.empty()) { std::cout << " Can be obtained at " << grid.url; } @@ -539,8 +541,9 @@ static void outputObject( // --------------------------------------------------------------------------- -static void outputOperationSummary(const CoordinateOperationNNPtr &op, - const DatabaseContextPtr &dbContext) { +static void outputOperationSummary( + const CoordinateOperationNNPtr &op, const DatabaseContextPtr &dbContext, + CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse) { auto ids = op->identifiers(); if (!ids.empty()) { std::cout << *(ids[0]->codeSpace()) << ":" << ids[0]->code(); @@ -586,10 +589,16 @@ static void outputOperationSummary(const CoordinateOperationNNPtr &op, if (dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); for (const auto &grid : setGrids) { if (!grid.available) { std::cout << ", at least one grid missing"; + if (gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE && + !grid.packageName.empty()) { + std::cout << " on the system, but available on CDN"; + } break; } } @@ -686,7 +695,7 @@ static void outputOperations( } if (summary) { for (const auto &op : list) { - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); } } else { bool first = true; @@ -701,7 +710,7 @@ static void outputOperations( "\xC2\xB0" << (i + 1) << ":" << std::endl << std::endl; - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); std::cout << std::endl; outputObject(dbContext, op, allowUseIntermediateCRS, outputOpt); } @@ -734,7 +743,9 @@ int main(int argc, char **argv) { CoordinateOperationContext::SourceTargetCRSExtentUse::SMALLEST; bool buildBoundCRSToWGS84 = false; CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse = - CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; + pj_context_is_network_enabled(nullptr) + ? CoordinateOperationContext::GridAvailabilityUse::KNOWN_AVAILABLE + : CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; CoordinateOperationContext::IntermediateCRSUse allowUseIntermediateCRS = CoordinateOperationContext::IntermediateCRSUse:: IF_NO_DIRECT_TRANSFORMATION; @@ -947,6 +958,9 @@ int main(int argc, char **argv) { } else if (ci_equal(value, "sort")) { gridAvailabilityUse = CoordinateOperationContext:: GridAvailabilityUse::USE_FOR_SORTING; + } else if (ci_equal(value, "known_available")) { + gridAvailabilityUse = CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE; } else { std::cerr << "Unrecognized value for option --grid-check: " << value << std::endl; diff --git a/src/filemanager.cpp b/src/filemanager.cpp index cd738d5e..d9a02632 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -69,7 +69,7 @@ NS_PROJ_START // --------------------------------------------------------------------------- -File::File() = default; +File::File(const std::string &name) : name_(name) {} // --------------------------------------------------------------------------- @@ -85,7 +85,8 @@ class FileStdio : public File { FileStdio &operator=(const FileStdio &) = delete; protected: - FileStdio(PJ_CONTEXT *ctx, FILE *fp) : m_ctx(ctx), m_fp(fp) {} + FileStdio(const std::string &name, PJ_CONTEXT *ctx, FILE *fp) + : File(name), m_ctx(ctx), m_fp(fp) {} public: ~FileStdio() override; @@ -130,7 +131,8 @@ unsigned long long FileStdio::tell() { std::unique_ptr FileStdio::open(PJ_CONTEXT *ctx, const char *filename) { auto fp = fopen(filename, "rb"); - return std::unique_ptr(fp ? new FileStdio(ctx, fp) : nullptr); + return std::unique_ptr(fp ? new FileStdio(filename, ctx, fp) + : nullptr); } // --------------------------------------------------------------------------- @@ -145,7 +147,8 @@ class FileLegacyAdapter : public File { FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; protected: - FileLegacyAdapter(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + FileLegacyAdapter(const std::string &name, PJ_CONTEXT *ctx, PAFile fp) + : File(name), m_ctx(ctx), m_fp(fp) {} public: ~FileLegacyAdapter() override; @@ -189,7 +192,7 @@ unsigned long long FileLegacyAdapter::tell() { std::unique_ptr FileLegacyAdapter::open(PJ_CONTEXT *ctx, const char *filename) { auto fid = pj_ctx_fopen(ctx, filename, "rb"); - return std::unique_ptr(fid ? new FileLegacyAdapter(ctx, fid) + return std::unique_ptr(fid ? new FileLegacyAdapter(filename, ctx, fid) : nullptr); } @@ -292,7 +295,7 @@ class NetworkFile : public File { PROJ_NETWORK_HANDLE *handle, unsigned long long lastDownloadOffset, unsigned long long filesize) - : m_ctx(ctx), m_url(url), m_handle(handle), + : File(url), m_ctx(ctx), m_url(url), m_handle(handle), m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} public: diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 949b223f..46391597 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -60,13 +60,16 @@ class FileManager { class File { protected: - File(); + std::string name_; + explicit File(const std::string &name); public: virtual ~File(); virtual size_t read(void *buffer, size_t sizeBytes) = 0; virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; virtual unsigned long long tell() = 0; + + const std::string &name() const { return name_; } }; //! @endcond Doxygen_Suppress diff --git a/src/grids.cpp b/src/grids.cpp index 91e51016..45e7e8b0 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -1299,14 +1299,15 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } - if (ends_with(filename, "gtx") || ends_with(filename, "GTX")) { - auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), filename); + const auto actualName(fp->name()); + if (ends_with(actualName, "gtx") || ends_with(actualName, "GTX")) { + auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), actualName); if (!grid) { return nullptr; } auto set = std::unique_ptr(new VerticalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "gtx"; set->m_grids.push_back(std::unique_ptr(grid)); return set; @@ -1324,7 +1325,7 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), filename); + auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), actualName); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; @@ -2130,6 +2131,7 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } + const auto actualName(fp->name()); char header[160]; /* -------------------------------------------------------------------- */ @@ -2151,35 +2153,35 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (header_size >= 144 + 16 && strncmp(header + 0, "HEADER", 6) == 0 && strncmp(header + 96, "W GRID", 6) == 0 && strncmp(header + 144, "TO NAD83 ", 16) == 0) { - auto grid = NTv1Grid::open(ctx, std::move(fp), filename); + auto grid = NTv1Grid::open(ctx, std::move(fp), actualName); if (!grid) { return nullptr; } auto set = std::unique_ptr( new HorizontalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "ntv1"; set->m_grids.push_back(std::unique_ptr(grid)); return set; } else if (header_size >= 9 && strncmp(header + 0, "CTABLE V2", 9) == 0) { - auto grid = CTable2Grid::open(ctx, std::move(fp), filename); + auto grid = CTable2Grid::open(ctx, std::move(fp), actualName); if (!grid) { return nullptr; } auto set = std::unique_ptr( new HorizontalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "ctable2"; set->m_grids.push_back(std::unique_ptr(grid)); return set; } else if (header_size >= 48 + 7 && strncmp(header + 0, "NUM_OREC", 8) == 0 && strncmp(header + 48, "GS_TYPE", 7) == 0) { - return NTv2GridSet::open(ctx, std::move(fp), filename); + return NTv2GridSet::open(ctx, std::move(fp), actualName); } else if (IsTIFF(header_size, reinterpret_cast(header))) { #ifdef TIFF_ENABLED - auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), filename); + auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), actualName); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; @@ -2450,6 +2452,7 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } + const auto actualName(fp->name()); /* -------------------------------------------------------------------- */ /* Load a header, to determine the file type. */ @@ -2463,7 +2466,8 @@ GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (IsTIFF(header_size, header)) { #ifdef TIFF_ENABLED - auto set = GTiffGenericGridShiftSet::open(ctx, std::move(fp), filename); + auto set = + GTiffGenericGridShiftSet::open(ctx, std::move(fp), actualName); if (!set) pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return set; diff --git a/src/iso19111/c_api.cpp b/src/iso19111/c_api.cpp index 9db9e5b3..5df4c513 100644 --- a/src/iso19111/c_api.cpp +++ b/src/iso19111/c_api.cpp @@ -177,7 +177,11 @@ static PJ *pj_obj_create(PJ_CONTEXT *ctx, const IdentifiedObjectNNPtr &objIn) { auto formatter = PROJStringFormatter::create( PROJStringFormatter::Convention::PROJ_5, dbContext); auto projString = coordop->exportToPROJString(formatter.get()); + if (pj_context_is_network_enabled(ctx)) { + ctx->defer_grid_opening = true; + } auto pj = pj_create_internal(ctx, projString.c_str()); + ctx->defer_grid_opening = false; if (pj) { pj->iso_obj = objIn; if (ctx->cpp_context) { @@ -766,7 +770,7 @@ int PROJ_DLL proj_grid_get_info_from_database( bool open_license; bool available; if (!db_context->lookForGridInfo( - grid_name, ctx->cpp_context->lastGridFullName_, + grid_name, false, ctx->cpp_context->lastGridFullName_, ctx->cpp_context->lastGridPackageName_, ctx->cpp_context->lastGridUrl_, direct_download, open_license, available)) { @@ -6571,7 +6575,10 @@ int proj_coordoperation_is_instantiable(PJ_CONTEXT *ctx, } auto dbContext = getDBcontextNoException(ctx, __FUNCTION__); try { - auto ret = op->isPROJInstantiable(dbContext) ? 1 : 0; + auto ret = op->isPROJInstantiable(dbContext, + pj_context_is_network_enabled(ctx)) + ? 1 + : 0; if (ctx->cpp_context) { ctx->cpp_context->autoCloseDbIfNeeded(); } @@ -6883,7 +6890,8 @@ int proj_coordoperation_get_grid_used_count(PJ_CONTEXT *ctx, try { if (!coordoperation->gridsNeededAsked) { coordoperation->gridsNeededAsked = true; - const auto gridsNeeded = co->gridsNeeded(dbContext); + const auto gridsNeeded = + co->gridsNeeded(dbContext, pj_context_is_network_enabled(ctx)); for (const auto &gridDesc : gridsNeeded) { coordoperation->gridsNeeded.emplace_back(gridDesc); } @@ -7220,6 +7228,12 @@ void PROJ_DLL proj_operation_factory_context_set_grid_availability_use( CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY); break; + + case PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + factory_ctx->operationContext->setGridAvailabilityUse( + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); + break; } } catch (const std::exception &e) { proj_log_error(ctx, __FUNCTION__, e.what()); diff --git a/src/iso19111/coordinateoperation.cpp b/src/iso19111/coordinateoperation.cpp index 6120c768..7c0515c7 100644 --- a/src/iso19111/coordinateoperation.cpp +++ b/src/iso19111/coordinateoperation.cpp @@ -788,13 +788,15 @@ void CoordinateOperation::setAccuracies( * available. */ bool CoordinateOperation::isPROJInstantiable( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { try { exportToPROJString(io::PROJStringFormatter::create().get()); } catch (const std::exception &) { return false; } - for (const auto &gridDesc : gridsNeeded(databaseContext)) { + for (const auto &gridDesc : + gridsNeeded(databaseContext, considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return false; } @@ -2013,8 +2015,9 @@ bool SingleOperation::_isEquivalentTo(const util::IComparable *other, // --------------------------------------------------------------------------- -std::set SingleOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set +SingleOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set res; for (const auto &genOpParamvalue : parameterValues()) { auto opParamvalue = dynamic_cast( @@ -2026,9 +2029,9 @@ std::set SingleOperation::gridsNeeded( desc.shortName = value->valueFile(); if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, - desc.url, desc.directDownload, desc.openLicense, - desc.available); + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, + desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); } @@ -10209,10 +10212,12 @@ bool ConcatenatedOperation::_isEquivalentTo( // --------------------------------------------------------------------------- std::set ConcatenatedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set res; for (const auto &operation : operations()) { - const auto l_gridsNeeded = operation->gridsNeeded(databaseContext); + const auto l_gridsNeeded = operation->gridsNeeded( + databaseContext, considerKnownGridsAsAvailable); for (const auto &gridDesc : l_gridsNeeded) { res.insert(gridDesc); } @@ -11132,7 +11137,10 @@ struct FilterResults { bool gridsKnown = true; if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { hasGrids = true; if (gridAvailabilityUse == @@ -11254,6 +11262,7 @@ struct FilterResults { CoordinateOperationPtr lastOp; bool first = true; + const auto gridAvailabilityUse = context->getGridAvailabilityUse(); for (const auto &op : res) { const auto curAccuracy = getAccuracy(op); bool dummy = false; @@ -11266,7 +11275,10 @@ struct FilterResults { if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { curHasGrids = true; curSetOfGrids.insert(gridDesc.shortName); @@ -11571,6 +11583,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11590,9 +11603,16 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( tmpAuthFactory->createFromCoordinateReferenceSystemCodes( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, false, context.extent1, context.extent2); res.insert(res.end(), resTmp.begin(), resTmp.end()); @@ -11635,6 +11655,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( std::list> ids; buildCRSIds(targetCRS, context, ids); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &id : ids) { const auto &targetAuthName = id.first; const auto &targetCode = id.second; @@ -11648,9 +11669,15 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( auto res = tmpAuthFactory->createFromCoordinateReferenceSystemCodes( std::string(), std::string(), targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, true, context.extent1, context.extent2); if (!res.empty()) { @@ -11698,6 +11725,7 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11717,21 +11745,28 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( std::vector res; if (useCreateBetweenGeodeticCRSWithDatumBasedIntermediates) { - res = tmpAuthFactory - ->createBetweenGeodeticCRSWithDatumBasedIntermediates( - sourceCRS, srcAuthName, srcCode, targetCRS, - targetAuthName, targetCode, - context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext:: - GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, - context.context->getDiscardSuperseded(), - authFactory->getAuthority() != "any" && - authorities.size() > 1 - ? authorities - : std::vector(), - context.extent1, context.extent2); + res = + tmpAuthFactory + ->createBetweenGeodeticCRSWithDatumBasedIntermediates( + sourceCRS, srcAuthName, srcCode, targetCRS, + targetAuthName, targetCode, + context.context->getUsePROJAlternativeGridNames(), + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + context.context->getDiscardSuperseded(), + authFactory->getAuthority() != "any" && + authorities.size() > 1 + ? authorities + : std::vector(), + context.extent1, context.extent2); } else { io::AuthorityFactory::ObjectType intermediateObjectType = io::AuthorityFactory::ObjectType::CRS; @@ -11749,9 +11784,15 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( res = tmpAuthFactory->createFromCRSCodesWithIntermediates( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), context.context->getIntermediateCRS(), intermediateObjectType, @@ -14167,15 +14208,21 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( componentsSrc[1], targetCRS->promoteTo3D(std::string(), dbContext), context); bool foundRegisteredTransformWithAllGridsAvailable = false; + const auto gridAvailabilityUse = + context.context->getGridAvailabilityUse(); const bool ignoreMissingGrids = - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY; for (const auto &op : verticalTransforms) { if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -14204,7 +14251,11 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -15034,8 +15085,9 @@ CoordinateOperationNNPtr PROJBasedOperation::_shallowClone() const { // --------------------------------------------------------------------------- -std::set PROJBasedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set +PROJBasedOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set res; try { @@ -15048,7 +15100,8 @@ std::set PROJBasedOperation::gridsNeeded( desc.shortName = shortName; if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, desc.url, + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); diff --git a/src/iso19111/factory.cpp b/src/iso19111/factory.cpp index 57850303..dae8680c 100644 --- a/src/iso19111/factory.cpp +++ b/src/iso19111/factory.cpp @@ -1018,14 +1018,14 @@ bool DatabaseContext::lookForGridAlternative(const std::string &officialName, // --------------------------------------------------------------------------- -bool DatabaseContext::lookForGridInfo(const std::string &projFilename, - std::string &fullFilename, - std::string &packageName, - std::string &url, bool &directDownload, - bool &openLicense, - bool &gridAvailable) const { +bool DatabaseContext::lookForGridInfo( + const std::string &projFilename, bool considerKnownGridsAsAvailable, + std::string &fullFilename, std::string &packageName, std::string &url, + bool &directDownload, bool &openLicense, bool &gridAvailable) const { Private::GridInfoCache info; - if (d->getGridInfoFromCache(projFilename, info)) { + const std::string key(projFilename + + (considerKnownGridsAsAvailable ? "true" : "false")); + if (d->getGridInfoFromCache(key, info)) { fullFilename = info.fullFilename; packageName = info.packageName; url = info.url; @@ -1041,16 +1041,20 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = false; directDownload = false; - fullFilename.resize(2048); - if (d->pjCtxt() == nullptr) { - d->setPjCtxt(pj_get_default_ctx()); + if (considerKnownGridsAsAvailable) { + fullFilename = projFilename; + } else { + fullFilename.resize(2048); + if (d->pjCtxt() == nullptr) { + d->setPjCtxt(pj_get_default_ctx()); + } + int errno_before = proj_context_errno(d->pjCtxt()); + gridAvailable = + pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], + fullFilename.size() - 1) != 0; + proj_context_errno_set(d->pjCtxt(), errno_before); + fullFilename.resize(strlen(fullFilename.c_str())); } - int errno_before = proj_context_errno(d->pjCtxt()); - gridAvailable = - pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], - fullFilename.size() - 1) != 0; - proj_context_errno_set(d->pjCtxt(), errno_before); - fullFilename.resize(strlen(fullFilename.c_str())); auto res = d->run("SELECT " @@ -1074,6 +1078,10 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = (row[3].empty() ? row[4] : row[3]) == "1"; directDownload = (row[5].empty() ? row[6] : row[5]) == "1"; + if (considerKnownGridsAsAvailable && !packageName.empty()) { + gridAvailable = true; + } + info.fullFilename = fullFilename; info.packageName = packageName; info.url = url; @@ -1082,7 +1090,7 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, } info.gridAvailable = gridAvailable; info.found = ret; - d->cache(projFilename, info); + d->cache(key, info); return ret; } @@ -1264,8 +1272,8 @@ struct AuthorityFactory::Private { return AuthorityFactory::create(context_, auth_name); } - bool - rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op); + bool rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable); UnitOfMeasure createUnitOfMeasure(const std::string &auth_name, const std::string &code); @@ -1392,8 +1400,10 @@ util::PropertyMap AuthorityFactory::Private::createProperties( // --------------------------------------------------------------------------- bool AuthorityFactory::Private::rejectOpDueToMissingGrid( - const operation::CoordinateOperationNNPtr &op) { - for (const auto &gridDesc : op->gridsNeeded(context())) { + const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable) { + for (const auto &gridDesc : + op->gridsNeeded(context(), considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return true; } @@ -3381,7 +3391,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSCode, const std::string &targetCRSCode) const { return createFromCoordinateReferenceSystemCodes( d->authority(), sourceCRSCode, d->authority(), targetCRSCode, false, - false, false); + false, false, false); } // --------------------------------------------------------------------------- @@ -3410,6 +3420,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param tryReverseOrder whether to search in the reverse order too (and thus @@ -3430,8 +3442,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, bool tryReverseOrder, - bool reportOnlyIntersectingTransformations, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + bool tryReverseOrder, bool reportOnlyIntersectingTransformations, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -3442,6 +3454,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( cacheKey += targetCRSCode; cacheKey += (usePROJAlternativeGridNames ? '1' : '0'); cacheKey += (discardIfMissingGrid ? '1' : '0'); + cacheKey += (considerKnownGridsAsAvailable ? '1' : '0'); cacheKey += (discardSuperseded ? '1' : '0'); cacheKey += (tryReverseOrder ? '1' : '0'); cacheKey += (reportOnlyIntersectingTransformations ? '1' : '0'); @@ -3680,7 +3693,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( target_crs_code != targetCRSCode))) { op = op->inverse(); } - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -3745,6 +3759,8 @@ static bool useIrrelevantPivot(const operation::CoordinateOperationNNPtr &op, * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param intermediateCRSAuthCodes List of (auth_name, code) of CRS that can be @@ -3773,7 +3789,7 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector> &intermediateCRSAuthCodes, ObjectType allowedIntermediateObjectType, @@ -4221,7 +4237,8 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( std::vector list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -4239,7 +4256,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, const std::vector &allowedAuthorities, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + const std::vector &allowedAuthorities, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -4822,7 +4840,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( std::vector list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 6bdd5510..926505ba 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -203,6 +203,27 @@ static const char *get_path_from_win32_projlib(const char *name, std::string& ou /* pj_open_lib_internal() */ /************************************************************************/ +#ifdef WIN32 +static const char dir_chars[] = "/\\"; +static const char dirSeparator = ';'; +#else +static const char dir_chars[] = "/"; +static const char dirSeparator = ':'; +#endif + +static bool is_tilde_slash(const char* name) +{ + return *name == '~' && strchr(dir_chars,name[1]); +} + +static bool is_rel_or_absolute_filename(const char *name) +{ + return strchr(dir_chars,*name) + || (*name == '.' && strchr(dir_chars,name[1])) + || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) + || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])); +} + static void* pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, void* (*open_file)(projCtx, const char*, const char*), @@ -211,13 +232,6 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, std::string fname; const char *sysname = nullptr; void* fid = nullptr; -#ifdef WIN32 - static const char dir_chars[] = "/\\"; - const char dirSeparator = ';'; -#else - static const char dir_chars[] = "/"; - const char dirSeparator = ':'; -#endif if( ctx == nullptr ) { ctx = pj_get_default_ctx(); @@ -227,7 +241,7 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, out_full_filename[0] = '\0'; /* check if ~/name */ - if (*name == '~' && strchr(dir_chars,name[1]) ) + if (is_tilde_slash(name)) if ((sysname = getenv("HOME")) != nullptr) { fname = sysname; fname += DIR_CHAR; @@ -236,11 +250,8 @@ pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, } else return nullptr; - /* or fixed path: /name, ./name or ../name */ - else if (strchr(dir_chars,*name) - || (*name == '.' && strchr(dir_chars,name[1])) - || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) - || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) + /* or fixed path: /name, ./name or ../name or http[s]:// */ + else if (is_rel_or_absolute_filename(name) || starts_with(name, "http://") || starts_with(name, "https://")) sysname = name; @@ -344,11 +355,31 @@ static void* pj_open_file_with_manager(projCtx ctx, const char *name, std::unique_ptr NS_PROJ::FileManager::open_resource_file( projCtx ctx, const char *name) { - return std::unique_ptr( + auto file = std::unique_ptr( reinterpret_cast( pj_open_lib_internal(ctx, name, "rb", pj_open_file_with_manager, nullptr, 0))); + if( file == nullptr && + !is_tilde_slash(name) && + !is_rel_or_absolute_filename(name) && + !starts_with(name, "http://") && + !starts_with(name, "https://") && + pj_context_is_network_enabled(ctx) ) { + std::string remote_file("https://cdn.proj.org/"); + remote_file += name; + auto pos = remote_file.rfind('.'); + if( pos + 4 == remote_file.size() ) { + remote_file = remote_file.substr(0, pos) + ".tif"; + file = open(ctx, remote_file.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } + } + } + return file; } /************************************************************************/ diff --git a/src/proj.h b/src/proj.h index 4f7878a4..7210318c 100644 --- a/src/proj.h +++ b/src/proj.h @@ -697,6 +697,12 @@ typedef enum { /** Ignore grid availability at all. Results will be presented as if * all grids were available. */ PROJ_GRID_AVAILABILITY_IGNORED, + + /** Results will be presented as if grids known to PROJ (that is + * registered in the grid_alternatives table of its database) were + * available. Used typically when networking is enabled. + */ + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE, } PROJ_GRID_AVAILABILITY_USE; /** \brief PROJ string version. */ diff --git a/src/proj_internal.h b/src/proj_internal.h index d54d8fb9..983f1c07 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -697,6 +697,7 @@ struct projCtx_t { void* file_finder_user_data = nullptr; projNetworkCallbacksAndData networking{}; + bool defer_grid_opening = false; // set by pj_obj_create() int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() @@ -826,7 +827,8 @@ PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); -bool pj_context_is_network_enabled(PJ_CONTEXT* ctx); +// For use by projinfo +bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); /* classic public API */ #include "proj_api.h" diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index e9983df6..3b6e366f 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -17,6 +17,7 @@ struct hgridshiftData { double t_final = 0; double t_epoch = 0; ListOfHGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace @@ -25,6 +26,14 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -40,6 +49,14 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -114,9 +131,9 @@ PJ *TRANSFORMATION(hgridshift,0) { return destructor (P, PJD_ERR_NO_ARGS); } - /* TODO: Refactor into shared function that can be used */ - /* by both vgridshift and hgridshift */ - if (pj_param(P->ctx, P->params, "tt_final").i) { + /* TODO: Refactor into shared function that can be used */ + /* by both vgridshift and hgridshift */ + if (pj_param(P->ctx, P->params, "tt_final").i) { Q->t_final = pj_param (P->ctx, P->params, "dt_final").f; if (Q->t_final == 0) { /* a number wasn't passed to +t_final, let's see if it was "now" */ @@ -131,16 +148,21 @@ PJ *TRANSFORMATION(hgridshift,0) { } } - if (pj_param(P->ctx, P->params, "tt_epoch").i) + if (pj_param(P->ctx, P->params, "tt_epoch").i) Q->t_epoch = pj_param (P->ctx, P->params, "dt_epoch").f; - Q->grids = proj_hgrid_init(P, "grids"); - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "hgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; } + else { + Q->grids = proj_hgrid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "hgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + } return P; } diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index b964f45b..f35832e1 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -18,14 +18,51 @@ struct vgridshiftData { double t_epoch = 0; double forward_multiplier = 0; ListOfVGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace +static void deal_with_vertcon_gtx_hack(PJ *P) +{ + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; + // The .gtx VERTCON files stored millimetres, but the .tif files + // are in metres. + if( Q->forward_multiplier != 0.001 ) { + return; + } + const char* gridname = pj_param(P->ctx, P->params, "sgrids").s; + if( !gridname ) { + return; + } + if( strcmp(gridname, "vertconw.gtx") != 0 && + strcmp(gridname, "vertconc.gtx") != 0 && + strcmp(gridname, "vertcone.gtx") != 0 ) { + return; + } + if( Q->grids.empty() ) { + return; + } + const auto& grids = Q->grids[0]->grids(); + if( !grids.empty() && + grids[0]->name().find(".tif") != std::string::npos ) { + Q->forward_multiplier = 1.0; + } +} + static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -41,6 +78,15 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -132,13 +178,18 @@ PJ *TRANSFORMATION(vgridshift,0) { Q->forward_multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; } - /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ - Q->grids = proj_vgrid_init(P, "grids"); - - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "vgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ + Q->grids = proj_vgrid_init(P, "grids"); + + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "vgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } P->fwd4d = forward_4d; diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index 23ff5076..ce11ae4e 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -68,6 +68,6 @@ test_network_CXXFLAGS = @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ test_network_LDADD = ../../src/libproj.la @GTEST_LIBS@ @CURL_LIBS@ test_network-check: test_network - PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network + PROJ_LIB=$(PROJ_LIB) PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check test_network-check diff --git a/test/unit/test_factory.cpp b/test/unit/test_factory.cpp index 93b2ef34..e342dad9 100644 --- a/test/unit/test_factory.cpp +++ b/test/unit/test_factory.cpp @@ -1617,34 +1617,34 @@ class FactoryWithTmpDatabase : public ::testing::Test { DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast(res[0])); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {std::make_pair(std::string("NS_PIVOT"), - std::string("PIVOT"))}); + false, false, {std::make_pair(std::string("NS_PIVOT"), + std::string("PIVOT"))}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast(res[0])); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {std::make_pair(std::string("NS_PIVOT"), - std::string("NOT_EXISTING"))}); + false, false, {std::make_pair(std::string("NS_PIVOT"), + std::string("NOT_EXISTING"))}); EXPECT_EQ(res.size(), 0U); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, + false, false, {std::make_pair(std::string("BAD_NS"), std::string("PIVOT"))}); EXPECT_EQ(res.size(), 0U); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_TARGET", "TARGET", "NS_SOURCE", "SOURCE", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast(res[0])); @@ -1654,7 +1654,7 @@ class FactoryWithTmpDatabase : public ::testing::Test { DatabaseContext::create(m_ctxt), std::string()); auto res = factory->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast(res[0])); @@ -1833,7 +1833,7 @@ TEST(factory, AuthorityFactory_createFromCoordinateReferenceSystemCodes) { { // Test removal of superseded transform auto list = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4179", "EPSG", "4258", false, false, true); + "EPSG", "4179", "EPSG", "4258", false, false, false, true); ASSERT_EQ(list.size(), 2U); // Romania has a larger area than Poland (given our approx formula) EXPECT_EQ(list[0]->getEPSGCode(), 15994); // Romania - 3m @@ -1851,12 +1851,12 @@ TEST( { auto res = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "32631", false, false, false); + "EPSG", "4326", "EPSG", "32631", false, false, false, false); ASSERT_EQ(res.size(), 1U); } { auto res = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4209", "EPSG", "4326", false, false, false); + "EPSG", "4209", "EPSG", "4326", false, false, false, false); EXPECT_TRUE(!res.empty()); for (const auto &conv : res) { EXPECT_TRUE(conv->sourceCRS()->getEPSGCode() == 4209); @@ -1889,7 +1889,8 @@ TEST_F(FactoryWithTmpDatabase, DatabaseContext::create(m_ctxt), std::string()); { auto res = factoryGeneral->createFromCoordinateReferenceSystemCodes( - "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false); + "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false, + false); ASSERT_EQ(res.size(), 1U); } @@ -1897,7 +1898,8 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory::create(DatabaseContext::create(m_ctxt), "EPSG"); { auto res = factoryEPSG->createFromCoordinateReferenceSystemCodes( - "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false); + "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false, + false); ASSERT_EQ(res.size(), 1U); } @@ -1919,17 +1921,17 @@ TEST_F(FactoryWithTmpDatabase, << last_error(); { auto res = factoryGeneral->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); } { auto res = factoryEPSG->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 0U); } { auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); } } @@ -1982,7 +1984,7 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 3U); EXPECT_EQ(*(res[0]->name()->description()), "TRANSFORMATION_1M"); EXPECT_EQ(*(res[1]->name()->description()), "TRANSFORMATION_10M"); @@ -2001,7 +2003,7 @@ TEST_F( auto factory = AuthorityFactory::create(DatabaseContext::create(m_ctxt), std::string()); auto res = factory->createFromCRSCodesWithIntermediates( - "EPSG", "4326", "EPSG", "4326", false, false, false, {}); + "EPSG", "4326", "EPSG", "4326", false, false, false, false, {}); EXPECT_EQ(res.size(), 0U); } @@ -2085,7 +2087,7 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory_proj_based_transformation) { auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); EXPECT_EQ(res[0]->nameStr(), "My PROJ string based op"); EXPECT_EQ(res[0]->exportToPROJString(PROJStringFormatter::create().get()), @@ -2146,7 +2148,7 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory_wkt_based_transformation) { auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); EXPECT_EQ(res[0]->nameStr(), "My WKT string based op"); EXPECT_EQ(res[0]->exportToPROJString(PROJStringFormatter::create().get()), @@ -2180,9 +2182,10 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); - EXPECT_THROW(factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false), - FactoryException); + EXPECT_THROW( + factoryOTHER->createFromCoordinateReferenceSystemCodes( + "EPSG", "4326", "EPSG", "4326", false, false, false, false), + FactoryException); } // --------------------------------------------------------------------------- @@ -2207,9 +2210,10 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); - EXPECT_THROW(factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false), - FactoryException); + EXPECT_THROW( + factoryOTHER->createFromCoordinateReferenceSystemCodes( + "EPSG", "4326", "EPSG", "4326", false, false, false, false), + FactoryException); } // --------------------------------------------------------------------------- @@ -2262,7 +2266,7 @@ TEST_F(FactoryWithTmpDatabase, lookForGridInfo) { bool openLicense = false; bool gridAvailable = false; EXPECT_TRUE(DatabaseContext::create(m_ctxt)->lookForGridInfo( - "PROJ_fake_grid", fullFilename, packageName, url, directDownload, + "PROJ_fake_grid", false, fullFilename, packageName, url, directDownload, openLicense, gridAvailable)); EXPECT_TRUE(fullFilename.empty()); EXPECT_TRUE(packageName.empty()); diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index ba592da4..c3372ca9 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -598,4 +598,90 @@ TEST(networking, getfilesize) { proj_context_destroy(ctx); } +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_hgridshift) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, 39.99999839, 1e-8); + EXPECT_NEAR(c.xyz.y, -79.99999807, 1e-8); + EXPECT_NEAR(c.xyz.z, 0, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_vgridshift) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // WGS84 to EGM2008 height. Using egm08_25.tif + auto P = + proj_create_crs_to_crs(ctx, "EPSG:4326", "EPSG:4326+3855", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = -30; // lat + c.xyz.y = 150; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, -30, 1e-8); + EXPECT_NEAR(c.xyz.y, 150, 1e-8); + EXPECT_NEAR(c.xyz.z, -31.89, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_vgridshift_vertcon) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // NGVD29 to NAVD88 height. Using vertcone.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269+7968", "EPSG:4269+5703", + nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, 40, 1e-8); + EXPECT_NEAR(c.xyz.y, -80, 1e-8); + EXPECT_NEAR(c.xyz.z, -0.15, 1e-2); +} + +#endif + } // namespace diff --git a/test/unit/test_operation.cpp b/test/unit/test_operation.cpp index 9cde8822..753878c5 100644 --- a/test/unit/test_operation.cpp +++ b/test/unit/test_operation.cpp @@ -4789,7 +4789,7 @@ TEST(operation, geogCRS_to_geogCRS_context_concatenated_operation) { EXPECT_TRUE(nn_dynamic_pointer_cast(list[0]) != nullptr); - auto grids = list[0]->gridsNeeded(DatabaseContext::create()); + auto grids = list[0]->gridsNeeded(DatabaseContext::create(), false); EXPECT_EQ(grids.size(), 1U); } @@ -6402,7 +6402,7 @@ TEST(operation, transformation_height_to_PROJ_string) { EXPECT_EQ(transf->exportToPROJString(PROJStringFormatter::create().get()), "+proj=vgridshift +grids=egm08_25.gtx +multiplier=1"); - auto grids = transf->gridsNeeded(DatabaseContext::create()); + auto grids = transf->gridsNeeded(DatabaseContext::create(), false); ASSERT_EQ(grids.size(), 1U); auto gridDesc = *(grids.begin()); EXPECT_EQ(gridDesc.shortName, "egm08_25.gtx"); @@ -6702,7 +6702,7 @@ TEST(operation, compoundCRS_with_boundGeogCRS_and_boundVerticalCRS_to_geogCRS) { "+step +proj=unitconvert +xy_in=rad +xy_out=deg " "+step +proj=axisswap +order=2,1"); - auto grids = op->gridsNeeded(DatabaseContext::create()); + auto grids = op->gridsNeeded(DatabaseContext::create(), false); EXPECT_EQ(grids.size(), 1U); auto opInverse = CoordinateOperationFactory::create()->createOperation( @@ -8128,8 +8128,8 @@ TEST(operation, isPROJInstantiable) { auto transformation = Transformation::createGeocentricTranslations( PropertyMap(), GeographicCRS::EPSG_4269, GeographicCRS::EPSG_4326, 1.0, 2.0, 3.0, {}); - EXPECT_TRUE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_TRUE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } // Missing grid @@ -8137,8 +8137,8 @@ TEST(operation, isPROJInstantiable) { auto transformation = Transformation::createNTv2( PropertyMap(), GeographicCRS::EPSG_4807, GeographicCRS::EPSG_4326, "foo.gsb", std::vector()); - EXPECT_FALSE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_FALSE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } // Unsupported method @@ -8149,8 +8149,8 @@ TEST(operation, isPROJInstantiable) { PropertyMap(), std::vector{}), std::vector{}, std::vector{}); - EXPECT_FALSE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_FALSE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } } -- cgit v1.2.3 From 2093aca0720949303410280912b61efd791d2f01 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 20:13:03 +0100 Subject: Network: make CDN endpoint configurable either in proj.ini, PROJ_NETWORK_ENDPOINT or proj_context_set_url_endpoint() --- data/Makefile.am | 4 +- data/proj.ini | 10 +++ docs/source/resource_files.rst | 24 +++++++ docs/source/usage/environmentvars.rst | 9 +++ scripts/reference_exported_symbols.txt | 1 + src/filemanager.cpp | 25 ++++++++ src/open_lib.cpp | 113 ++++++++++++++++++++++++++++++--- src/proj.h | 2 + src/proj_internal.h | 7 ++ test/unit/CMakeLists.txt | 9 ++- test/unit/test_network.cpp | 55 ++++++++++++++++ 11 files changed, 245 insertions(+), 14 deletions(-) create mode 100644 data/proj.ini diff --git a/data/Makefile.am b/data/Makefile.am index 3778a2be..09fa1989 100644 --- a/data/Makefile.am +++ b/data/Makefile.am @@ -1,6 +1,6 @@ DATAPATH = $(top_srcdir)/data -pkgdata_DATA = GL27 nad.lst nad27 nad83 world other.extra \ +pkgdata_DATA = proj.ini GL27 nad.lst nad27 nad83 world other.extra \ CH null \ ITRF2000 ITRF2008 ITRF2014 proj.db \ projjson.schema.json @@ -38,7 +38,7 @@ SQL_ORDERED_LIST = sql/begin.sql \ sql/customizations.sql \ sql/commit.sql -EXTRA_DIST = GL27 nad.lst nad27 nad83 \ +EXTRA_DIST = proj.ini GL27 nad.lst nad27 nad83 \ world other.extra \ CH \ ITRF2000 ITRF2008 ITRF2014 \ diff --git a/data/proj.ini b/data/proj.ini new file mode 100644 index 00000000..f42bc940 --- /dev/null +++ b/data/proj.ini @@ -0,0 +1,10 @@ +[general] +; Lines starting by ; are commented lines. +; + +; Network capabilities disabled by default. +; Can be overriden with the PROJ_NETWORK=ON environment variable. +; network = on + +; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. +cdn_endpoint = https://cdn.proj.org diff --git a/docs/source/resource_files.rst b/docs/source/resource_files.rst index 69cf2f95..ea02fd4b 100644 --- a/docs/source/resource_files.rst +++ b/docs/source/resource_files.rst @@ -54,6 +54,30 @@ A proj installation includes a SQLite database of transformation information that must be accessible for the library to work properly. The library will print an error if the database can't be found. +proj.ini +------------------------------------------------------------------------------- + +.. versionadded:: 7.0 + +proj.ini is a text configuration file, mostly dedicated at setting up network +related parameters. + +Its default content is: + +:: + + [general] + ; Lines starting by ; are commented lines. + ; + + ; Network capabilities disabled by default. + ; Can be overriden with the PROJ_NETWORK=ON environment variable. + ; network = on + + ; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. + cdn_endpoint = https://cdn.proj.org + + Transformation grids ------------------------------------------------------------------------------- diff --git a/docs/source/usage/environmentvars.rst b/docs/source/usage/environmentvars.rst index 00058716..24ae4a45 100644 --- a/docs/source/usage/environmentvars.rst +++ b/docs/source/usage/environmentvars.rst @@ -65,3 +65,12 @@ done by setting the variable with no content:: (Content Delivery Network) storage, when grids are not available locally. Alternatively, the :c:func:`proj_context_set_enable_network` function can be used. + +.. envvar:: PROJ_NETWORK_ENDPOINT + + .. versionadded:: 7.0.0 + + Define the endpoint of the CDN storage. Normally defined through the proj.ini + configuration file locale in PROJ_LIB. + Alternatively, the :c:func:`proj_context_set_url_endpoint` function can + be used. diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index d2724825..3e17a3c7 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -800,6 +800,7 @@ proj_context_set_file_finder proj_context_set_network_callbacks proj_context_set(PJconsts*, projCtx_t*) proj_context_set_search_paths +proj_context_set_url_endpoint proj_context_use_proj4_init_rules proj_convert_conversion_to_other_method proj_coord diff --git a/src/filemanager.cpp b/src/filemanager.cpp index d9a02632..97d7369e 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -772,6 +772,9 @@ int proj_context_set_network_callbacks( /** Enable or disable network access. * +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK environment variable. +* * @param ctx PROJ context, or NULL * @param enable TRUE if network access is allowed. * @return TRUE if network access is possible. That is either libcurl is @@ -781,6 +784,8 @@ int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); ctx->networking.enabled_env_variable_checked = true; ctx->networking.enabled = enable != FALSE; #ifdef CURL_ENABLED @@ -793,6 +798,25 @@ int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { // --------------------------------------------------------------------------- +/** Define the URL endpoint to query for remote grids. +* +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK_ENDPOINT environment variable. +* +* @param ctx PROJ context, or NULL +* @param url Endpoint URL. Must NOT be NULL. +*/ +void proj_context_set_url_endpoint(PJ_CONTEXT *ctx, const char *url) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); + ctx->endpoint = url; +} + +// --------------------------------------------------------------------------- + //! @cond Doxygen_Suppress bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { @@ -808,6 +832,7 @@ bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { ci_equal(enabled, "YES") || ci_equal(enabled, "TRUE"); } + pj_load_ini(ctx); ctx->networking.enabled_env_variable_checked = true; return ctx->networking.enabled; } diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 926505ba..cde5be7b 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -366,16 +366,21 @@ std::unique_ptr NS_PROJ::FileManager::open_resource_file( !starts_with(name, "http://") && !starts_with(name, "https://") && pj_context_is_network_enabled(ctx) ) { - std::string remote_file("https://cdn.proj.org/"); - remote_file += name; - auto pos = remote_file.rfind('.'); - if( pos + 4 == remote_file.size() ) { - remote_file = remote_file.substr(0, pos) + ".tif"; - file = open(ctx, remote_file.c_str()); - if( file ) { - pj_log( ctx, PJ_LOG_DEBUG_MAJOR, - "Using %s", remote_file.c_str() ); - pj_ctx_set_errno( ctx, 0 ); + std::string remote_file(pj_context_get_url_endpoint(ctx)); + if( !remote_file.empty() ) { + if( remote_file.back() != '/' ) { + remote_file += '/'; + } + remote_file += name; + auto pos = remote_file.rfind('.'); + if( pos + 4 == remote_file.size() ) { + remote_file = remote_file.substr(0, pos) + ".tif"; + file = open(ctx, remote_file.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } } } } @@ -433,3 +438,91 @@ int pj_find_file(projCtx ctx, const char *short_filename, } return 0; } + +/************************************************************************/ +/* pj_context_get_url_endpoint() */ +/************************************************************************/ + +std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx) +{ + if( !ctx->endpoint.empty() ) { + return ctx->endpoint; + } + pj_load_ini(ctx); + return ctx->endpoint; +} + +/************************************************************************/ +/* trim() */ +/************************************************************************/ + +static std::string trim(const std::string& s) { + const auto first = s.find_first_not_of(' '); + const auto last = s.find_last_not_of(' '); + if( first == std::string::npos || last == std::string::npos ) { + return std::string(); + } + return s.substr(first, last - first + 1); +} + +/************************************************************************/ +/* pj_load_ini() */ +/************************************************************************/ + +void pj_load_ini(projCtx ctx) +{ + if( ctx->iniFileLoaded ) + return; + + const char* endpoint_from_env = getenv("PROJ_NETWORK_ENDPOINT"); + if( endpoint_from_env && endpoint_from_env[0] != '\0' ) { + ctx->endpoint = endpoint_from_env; + } + + ctx->iniFileLoaded = true; + auto file = std::unique_ptr( + reinterpret_cast( + pj_open_lib_internal(ctx, "proj.ini", "rb", + pj_open_file_with_manager, + nullptr, 0))); + if( !file ) + return; + file->seek(0, SEEK_END); + const auto filesize = file->tell(); + if( filesize == 0 || filesize > 100 * 1024U ) + return; + file->seek(0, SEEK_SET); + std::string content; + content.resize(static_cast(filesize)); + const auto nread = file->read(&content[0], content.size()); + if( nread != content.size() ) + return; + content += '\n'; + size_t pos = 0; + while( pos != std::string::npos ) { + const auto eol = content.find_first_of("\r\n", pos); + if( eol == std::string::npos ) { + break; + } + + const auto equal = content.find('=', pos); + if( equal < eol ) + { + const auto key = trim(content.substr(pos, equal-pos)); + const auto value = trim(content.substr(equal + 1, + eol - (equal+1))); + if( ctx->endpoint.empty() && key == "cdn_endpoint" ) { + ctx->endpoint = value; + } else if ( key == "network" ) { + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled == nullptr || enabled[0] == '\0') { + ctx->networking.enabled = ci_equal(value, "ON") || + ci_equal(value, "YES") || + ci_equal(value, "TRUE"); + } + } + } + + pos = content.find_first_not_of("\r\n", eol); + } +} diff --git a/src/proj.h b/src/proj.h index 7210318c..3f0879bb 100644 --- a/src/proj.h +++ b/src/proj.h @@ -424,6 +424,8 @@ int PROJ_DLL proj_context_set_network_callbacks( int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, int enabled); +void PROJ_DLL proj_context_set_url_endpoint(PJ_CONTEXT* ctx, const char* url); + /*! @cond Doxygen_Suppress */ /* Manage the transformation definition object PJ */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 983f1c07..669fd2b5 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -699,6 +699,9 @@ struct projCtx_t { projNetworkCallbacksAndData networking{}; bool defer_grid_opening = false; // set by pj_obj_create() + bool iniFileLoaded = false; + std::string endpoint{}; + int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() projCtx_t() = default; @@ -830,6 +833,10 @@ void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); // For use by projinfo bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); +std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx); + +void pj_load_ini(PJ_CONTEXT* ctx); + /* classic public API */ #include "proj_api.h" diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 23d54663..0dcafc85 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -160,5 +160,10 @@ target_link_libraries(test_network GTest::gtest ${PROJ_LIBRARIES}) add_test(NAME test_network COMMAND test_network) -set_property(TEST test_network - PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") +if(MSVC) + set_property(TEST test_network + PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") +else() + set_property(TEST test_network + PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") +endif() diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index c3372ca9..3e64329e 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -684,4 +684,59 @@ TEST(networking, curl_vgridshift_vertcon) { #endif +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, network_endpoint_env_variable) { + putenv(const_cast("PROJ_NETWORK_ENDPOINT=http://0.0.0.0/")); + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + putenv(const_cast("PROJ_NETWORK_ENDPOINT=")); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_EQ(c.xyz.x, HUGE_VAL); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, network_endpoint_api) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + proj_context_set_url_endpoint(ctx, "http://0.0.0.0"); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_EQ(c.xyz.x, HUGE_VAL); +} + +#endif + } // namespace -- cgit v1.2.3 From aa8c7826cf17e650ee2c3a2281aca49db37c4e81 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Wed, 25 Dec 2019 22:16:28 +0100 Subject: Network: rework error handling --- src/filemanager.cpp | 95 +++++++++++++++------ src/proj.h | 18 ++-- src/proj_internal.h | 1 - test/unit/test_network.cpp | 209 ++++++++++++++++++++++++++++++++++++--------- 4 files changed, 252 insertions(+), 71 deletions(-) diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 97d7369e..551301c6 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -323,11 +323,19 @@ std::unique_ptr NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { } else { std::vector buffer(DOWNLOAD_CHUNK_SIZE); size_t size_read = 0; - auto handle = - ctx->networking.open(ctx, filename, 0, buffer.size(), &buffer[0], - &size_read, ctx->networking.user_data); + std::string errorBuffer; + errorBuffer.resize(1024); + + auto handle = ctx->networking.open( + ctx, filename, 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); buffer.resize(size_read); gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename, + errorBuffer.c_str()); + } unsigned long long filesize = 0; if (handle) { @@ -405,11 +413,14 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); size_t nRead = 0; + std::string errorBuffer; + errorBuffer.resize(1024); if (!m_handle) { m_handle = m_ctx->networking.open( m_ctx, m_url.c_str(), offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], - &nRead, m_ctx->networking.user_data); + &nRead, errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); if (!m_handle) { return 0; } @@ -417,9 +428,15 @@ size_t NetworkFile::read(void *buffer, size_t sizeBytes) { nRead = m_ctx->networking.read_range( m_ctx, m_handle, offsetToDownload, m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + errorBuffer.size(), &errorBuffer[0], m_ctx->networking.user_data); } if (nRead == 0) { + errorBuffer.resize(strlen(errorBuffer.data())); + if (!errorBuffer.empty()) { + pj_log(m_ctx, PJ_LOG_ERROR, "Cannot read in %s: %s", + m_url.c_str(), errorBuffer.c_str()); + } return 0; } region.resize(nRead); @@ -542,10 +559,10 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- -static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, - unsigned long long offset, - size_t size_to_read, void *buffer, - size_t *out_size_read, void *) { +static PROJ_NETWORK_HANDLE * +pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *) { CURL *hCurlHandle = curl_easy_init(); if (!hCurlHandle) return nullptr; @@ -583,6 +600,10 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -594,10 +615,24 @@ static PROJ_NETWORK_HANDLE *pj_curl_open(PJ_CONTEXT *, const char *url, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } curl_easy_cleanup(hCurlHandle); return nullptr; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -619,7 +654,8 @@ static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, unsigned long long offset, size_t size_to_read, - void *buffer, void *) { + void *buffer, size_t error_string_max_size, + char *out_error_string, void *) { auto handle = reinterpret_cast(raw_handle); auto hCurlHandle = handle->m_handle; @@ -633,6 +669,10 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + szCurlErrBuf[0] = '\0'; + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + curl_easy_perform(hCurlHandle); long response_code = 0; @@ -641,9 +681,23 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } return 0; } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } if (!body.empty()) { memcpy(buffer, body.data(), std::min(size_to_read, body.size())); @@ -682,8 +736,12 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, size_t, /* size to read */ void *, /* buffer to update with bytes read*/ size_t *, /* output: size actually read */ + size_t error_string_max_size, char *out_error_string, void * /*user_data*/) { - pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Network functionality not available"); + if (out_error_string) { + snprintf(out_error_string, error_string_max_size, "%s", + "Network functionality not available"); + } return nullptr; } @@ -692,14 +750,6 @@ no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void * /*user_data*/) {} -// --------------------------------------------------------------------------- - -static const char *no_op_network_get_last_error(PJ_CONTEXT *, - PROJ_NETWORK_HANDLE *, - void * /*user_data*/) { - return "Network functionality not available"; -} - #endif // --------------------------------------------------------------------------- @@ -713,7 +763,6 @@ void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { #else ctx->networking.open = no_op_network_open; ctx->networking.close = no_op_network_close; - ctx->networking.get_last_error = no_op_network_get_last_error; #endif } @@ -741,7 +790,6 @@ NS_PROJ_END * @param close_cbk Callback to close a remote file. * @param get_header_value_cbk Callback to get HTTP headers * @param read_range_cbk Callback to read a range of bytes inside a remote file. - * @param get_last_error_cbk Callback to get last error message. * @param user_data Arbitrary pointer provided by the user, and passed to the * above callbacks. May be NULL. * @return TRUE in case of success. @@ -750,20 +798,17 @@ int proj_context_set_network_callbacks( PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, - proj_network_read_range_type read_range_cbk, - proj_network_get_last_error_type get_last_error_cbk, void *user_data) { + proj_network_read_range_type read_range_cbk, void *user_data) { if (ctx == nullptr) { ctx = pj_get_default_ctx(); } - if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk || - !get_last_error_cbk) { + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk) { return false; } ctx->networking.open = open_cbk; ctx->networking.close = close_cbk; ctx->networking.get_header_value = get_header_value_cbk; ctx->networking.read_range = read_range_cbk; - ctx->networking.get_last_error = get_last_error_cbk; ctx->networking.user_data = user_data; return true; } diff --git a/src/proj.h b/src/proj.h index 3f0879bb..96b9c3f8 100644 --- a/src/proj.h +++ b/src/proj.h @@ -369,6 +369,9 @@ typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; * headers from the server response to be able to respond to * proj_network_get_header_value_cbk_type callback. * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * * @return a non-NULL opaque handle in case of success. */ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( @@ -378,6 +381,8 @@ typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( size_t size_to_read, void* buffer, size_t* out_size_read, + size_t error_string_max_size, + char* out_error_string, void* user_data); /** Network access: close callback */ @@ -396,6 +401,10 @@ typedef const char* (*proj_network_get_header_value_cbk_type)( * * Read size_to_read bytes from handle, starting at offset, into * buffer. + * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * * @return the number of bytes actually read (0 in case of error) */ typedef size_t (*proj_network_read_range_type)( @@ -404,12 +413,8 @@ typedef size_t (*proj_network_read_range_type)( unsigned long long offset, size_t size_to_read, void* buffer, - void* user_data); - -/** Network access: get last error message */ -typedef const char* (*proj_network_get_last_error_type)( - PJ_CONTEXT* ctx, - PROJ_NETWORK_HANDLE*, + size_t error_string_max_size, + char* out_error_string, void* user_data); int PROJ_DLL proj_context_set_network_callbacks( @@ -418,7 +423,6 @@ int PROJ_DLL proj_context_set_network_callbacks( proj_network_close_cbk_type close_cbk, proj_network_get_header_value_cbk_type get_header_value_cbk, proj_network_read_range_type read_range_cbk, - proj_network_get_last_error_type get_last_error_cbk, void* user_data); int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, diff --git a/src/proj_internal.h b/src/proj_internal.h index 669fd2b5..27b30954 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -674,7 +674,6 @@ struct projNetworkCallbacksAndData proj_network_close_cbk_type close = nullptr; proj_network_get_header_value_cbk_type get_header_value = nullptr; proj_network_read_range_type read_range = nullptr; - proj_network_get_last_error_type get_last_error = nullptr; void* user_data = nullptr; }; diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index 3e64329e..89d66797 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -174,6 +174,7 @@ struct OpenEvent : public Event { unsigned long long offset = 0; size_t size_to_read = 0; std::vector response{}; + std::string errorMsg{}; int file_id = 0; }; @@ -197,13 +198,10 @@ struct ReadRangeEvent : public Event { unsigned long long offset = 0; size_t size_to_read = 0; std::vector response{}; + std::string errorMsg{}; int file_id = 0; }; -struct GetLastErrorEvent : public Event { - GetLastErrorEvent() { type = "GetLastErrorEvent"; } -}; - struct File {}; struct ExchangeWithCallback { @@ -220,7 +218,9 @@ struct ExchangeWithCallback { static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url, unsigned long long offset, size_t size_to_read, void *buffer, - size_t *out_size_read, void *user_data) { + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *user_data) { auto exchange = static_cast(user_data); if (exchange->error) return nullptr; @@ -251,9 +251,14 @@ static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url, exchange->error = true; return nullptr; } + if (!openEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + openEvent->errorMsg.c_str()); + return nullptr; + } + memcpy(buffer, openEvent->response.data(), openEvent->response.size()); *out_size_read = openEvent->response.size(); - auto handle = reinterpret_cast(new File()); exchange->mapIdToHandle[openEvent->file_id] = handle; return handle; @@ -336,7 +341,8 @@ static const char *get_header_value_cbk(PJ_CONTEXT *ctx, static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, unsigned long long offset, size_t size_to_read, - void *buffer, void *user_data) { + void *buffer, size_t error_string_max_size, + char *out_error_string, void *user_data) { auto exchange = static_cast(user_data); if (exchange->error) return 0; @@ -369,43 +375,24 @@ static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, exchange->error = true; return 0; } + exchange->nextEvent++; + if (!readRangeEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + readRangeEvent->errorMsg.c_str()); + return 0; + } memcpy(buffer, readRangeEvent->response.data(), readRangeEvent->response.size()); - exchange->nextEvent++; return readRangeEvent->response.size(); } -static const char *get_last_error_cbk(PJ_CONTEXT * /*ctx*/, - PROJ_NETWORK_HANDLE *, void *user_data) { - auto exchange = static_cast(user_data); - if (exchange->error) - return ""; - if (exchange->nextEvent >= exchange->events.size()) { - fprintf(stderr, "unexpected call to get_last_error()\n"); - exchange->error = true; - return ""; - } - auto getLastErrorEvent = dynamic_cast( - exchange->events[exchange->nextEvent].get()); - if (!getLastErrorEvent) { - fprintf( - stderr, - "unexpected call to get_last_error(). Was expecting a %s event\n", - exchange->events[exchange->nextEvent]->type.c_str()); - exchange->error = true; - return ""; - } - exchange->nextEvent++; - return ""; -} - TEST(networking, custom) { auto ctx = proj_context_create(); proj_context_set_enable_network(ctx, true); ExchangeWithCallback exchange; - ASSERT_TRUE(proj_context_set_network_callbacks( - ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, - get_last_error_cbk, &exchange)); + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); { std::unique_ptr event(new OpenEvent()); @@ -539,9 +526,9 @@ TEST(networking, getfilesize) { auto ctx = proj_context_create(); proj_context_set_enable_network(ctx, true); ExchangeWithCallback exchange; - ASSERT_TRUE(proj_context_set_network_callbacks( - ctx, open_cbk, close_cbk, get_header_value_cbk, read_range_cbk, - get_last_error_cbk, &exchange)); + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); { std::unique_ptr event(new OpenEvent()); @@ -600,6 +587,152 @@ TEST(networking, getfilesize) { // --------------------------------------------------------------------------- +TEST(networking, simul_open_error) { + auto ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/open_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->errorMsg = "Cannot open file"; + event->file_id = 1; + + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/open_error.tif +multiplier=1"); + + ASSERT_EQ(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, simul_read_range_error) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "dummy"; // dummy value: not used + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create(ctx, "+proj=vgridshift " + "+grids=https://foo/read_range_error.tif " + "+multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr event(new ReadRangeEvent()); + event->ctx = ctx; + event->offset = 3670016; + event->size_to_read = 278528; + event->errorMsg = "read range error"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = -49 / 180. * M_PI; + double z = 0; + proj_log_func(ctx, nullptr, silent_logger); + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, HUGE_VAL); + } + { + std::unique_ptr event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + #ifdef CURL_ENABLED TEST(networking, curl_hgridshift) { -- cgit v1.2.3 From f085b39752d00a296c288be42dfc69b39b73823f Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 12:25:03 +0100 Subject: Handle context reassignment for Grid/GridSet/File objects --- src/ctx.cpp | 4 +- src/filemanager.cpp | 18 +++++++- src/filemanager.hpp | 1 + src/grids.cpp | 88 ++++++++++++++++++++++++++++++++++++ src/grids.hpp | 12 +++++ src/pipeline.cpp | 4 +- src/proj_internal.h | 4 +- src/transformations/hgridshift.cpp | 9 ++++ src/transformations/vgridshift.cpp | 10 ++++ src/transformations/xyzgridshift.cpp | 10 ++++ test/unit/test_network.cpp | 4 ++ 11 files changed, 156 insertions(+), 8 deletions(-) diff --git a/src/ctx.cpp b/src/ctx.cpp index a7cf8cb3..6172b3c8 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -61,9 +61,9 @@ void pj_set_ctx( projPJ pj, projCtx ctx ) if (pj==nullptr) return; pj->ctx = ctx; - if( pj->is_pipeline ) + if( pj->reassign_context ) { - pj_pipeline_assign_context_to_steps(pj, ctx); + pj->reassign_context(pj, ctx); } for( const auto &alt: pj->alternativeCoordinateOperations ) { diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 551301c6..dabb46e0 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -94,6 +94,7 @@ class FileStdio : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence = SEEK_SET) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -156,6 +157,7 @@ class FileLegacyAdapter : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence = SEEK_SET) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -286,6 +288,7 @@ class NetworkFile : public File { size_t m_nBlocksToDownload = 1; unsigned long long m_lastDownloadedOffset; unsigned long long m_filesize; + proj_network_close_cbk_type m_closeCbk; NetworkFile(const NetworkFile &) = delete; NetworkFile &operator=(const NetworkFile &) = delete; @@ -296,7 +299,8 @@ class NetworkFile : public File { unsigned long long lastDownloadOffset, unsigned long long filesize) : File(url), m_ctx(ctx), m_url(url), m_handle(handle), - m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize) {} + m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize), + m_closeCbk(ctx->networking.close) {} public: ~NetworkFile() override; @@ -304,6 +308,7 @@ class NetworkFile : public File { size_t read(void *buffer, size_t sizeBytes) override; bool seek(unsigned long long offset, int whence) override; unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override; static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename); }; @@ -500,6 +505,17 @@ NetworkFile::~NetworkFile() { // --------------------------------------------------------------------------- +void NetworkFile::reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + if (m_closeCbk != m_ctx->networking.close) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Networking close callback has changed following context " + "reassignment ! This is highly suspicious"); + } +} + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifndef REMOVE_LEGACY_SUPPORT // If the user has specified a legacy fileapi, use it diff --git a/src/filemanager.hpp b/src/filemanager.hpp index 46391597..972634c2 100644 --- a/src/filemanager.hpp +++ b/src/filemanager.hpp @@ -68,6 +68,7 @@ class File { virtual size_t read(void *buffer, size_t sizeBytes) = 0; virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; virtual unsigned long long tell() = 0; + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; const std::string &name() const { return name_; } }; diff --git a/src/grids.cpp b/src/grids.cpp index 45e7e8b0..a3d984de 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -139,6 +139,7 @@ class NullVerticalShiftGrid : public VerticalShiftGrid { bool isNullGrid() const override { return true; } bool valueAt(int, int, float &out) const override; bool isNodata(float, double) const override { return false; } + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- @@ -171,6 +172,11 @@ class GTXVerticalShiftGrid : public VerticalShiftGrid { static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &name); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -412,6 +418,8 @@ class GTiffGrid : public Grid { std::string metadataItem(const std::string &key, int sample = -1) const; uint32 subfileType() const { return m_subfileType; } + + void reassign_context(PJ_CONTEXT *ctx) { m_ctx = ctx; } }; // --------------------------------------------------------------------------- @@ -738,6 +746,11 @@ class GTiffDataset { bool openTIFF(const std::string &filename); std::unique_ptr nextGrid(); + + void reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1056,6 +1069,11 @@ class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + VerticalShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } }; #endif // TIFF_ENABLED @@ -1150,6 +1168,10 @@ class GTiffVGrid : public VerticalShiftGrid { } void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1374,6 +1396,14 @@ const VerticalShiftGrid *VerticalShiftGridSet::gridAt(double lon, // --------------------------------------------------------------------------- +void VerticalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + HorizontalShiftGrid::HorizontalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) @@ -1402,6 +1432,8 @@ class NullHorizontalShiftGrid : public HorizontalShiftGrid { bool isNullGrid() const override { return true; } bool valueAt(int, int, float &lonShift, float &latShift) const override; + + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- @@ -1443,6 +1475,11 @@ class NTv1Grid : public HorizontalShiftGrid { static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1556,6 +1593,11 @@ class CTable2Grid : public HorizontalShiftGrid { static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1654,6 +1696,11 @@ class NTv2GridSet : public HorizontalShiftGridSet { static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1679,6 +1726,11 @@ class NTv2Grid : public HorizontalShiftGrid { m_mustSwap(mustSwapIn) {} bool valueAt(int, int, float &lonShift, float &latShift) const override; + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1857,6 +1909,11 @@ class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -1884,6 +1941,10 @@ class GTiffHGrid : public HorizontalShiftGrid { bool valueAt(int x, int y, float &lonShift, float &latShift) const override; void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -2235,6 +2296,14 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, return nullptr; } +// --------------------------------------------------------------------------- + +void HorizontalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + #ifdef TIFF_ENABLED // --------------------------------------------------------------------------- @@ -2250,6 +2319,11 @@ class GTiffGenericGridShiftSet : public GenericShiftGridSet, static std::unique_ptr open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + GenericShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -2287,6 +2361,10 @@ class GTiffGenericGrid : public GenericShiftGrid { void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -2356,6 +2434,8 @@ class NullGenericShiftGrid : public GenericShiftGrid { std::string metadataItem(const std::string &, int) const override { return std::string(); } + + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- @@ -2516,6 +2596,14 @@ const GenericShiftGrid *GenericShiftGridSet::gridAt(double lon, // --------------------------------------------------------------------------- +void GenericShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { std::string key("s"); key += gridkey; diff --git a/src/grids.hpp b/src/grids.hpp index 0f595754..fde3eb3e 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -88,6 +88,8 @@ class VerticalShiftGrid : public Grid { // x = 0 is western-most column, y = 0 is southern-most line virtual bool valueAt(int x, int y, float &out) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- @@ -112,6 +114,8 @@ class VerticalShiftGridSet { return m_grids; } const VerticalShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -130,6 +134,8 @@ class HorizontalShiftGrid : public Grid { // x = 0 is western-most column, y = 0 is southern-most line virtual bool valueAt(int x, int y, float &lonShift, float &latShift) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- @@ -154,6 +160,8 @@ class HorizontalShiftGridSet { return m_grids; } const HorizontalShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -181,6 +189,8 @@ class GenericShiftGrid : public Grid { // x = 0 is western-most column, y = 0 is southern-most line virtual bool valueAt(int x, int y, int sample, float &out) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- @@ -205,6 +215,8 @@ class GenericShiftGridSet { return m_grids; } const GenericShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- diff --git a/src/pipeline.cpp b/src/pipeline.cpp index 96767143..f65dbfa0 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -155,7 +155,7 @@ static PJ_LPZ pipeline_reverse_3d (PJ_XYZ xyz, PJ *P); static PJ_XY pipeline_forward (PJ_LP lp, PJ *P); static PJ_LP pipeline_reverse (PJ_XY xy, PJ *P); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ) +static void pipeline_reassign_context( PJ* P, PJ_CONTEXT* ctx ) { auto pipeline = static_cast(P->opaque); for( auto& step: pipeline->steps ) @@ -413,7 +413,7 @@ PJ *OPERATION(pipeline,0) { P->fwd = pipeline_forward; P->inv = pipeline_reverse; P->destructor = destructor; - P->is_pipeline = 1; + P->reassign_context = pipeline_reassign_context; /* Currently, the pipeline driver is a raw bit mover, enabling other operations */ /* to collaborate efficiently. All prep/fin stuff is done at the step levels. */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 27b30954..12ada034 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -343,6 +343,7 @@ struct PJconsts { PJ_OPERATOR inv4d = nullptr; PJ_DESTRUCTOR destructor = nullptr; + void (*reassign_context)(PJ*, projCtx_t *) = nullptr; /************************************************************************************* @@ -410,7 +411,6 @@ struct PJconsts { int geoc = 0; /* Geocentric latitude flag */ int is_latlong = 0; /* proj=latlong ... not really a projection at all */ int is_geocent = 0; /* proj=geocent ... not really a projection at all */ - int is_pipeline = 0; /* 1 if PJ represents a pipeline */ int need_ellps = 0; /* 0 for operations that are purely cartesian */ int skip_fwd_prepare = 0; int skip_fwd_finalize = 0; @@ -827,8 +827,6 @@ std::string pj_double_quote_string_param_if_needed(const std::string& str); PJ *pj_create_internal (PJ_CONTEXT *ctx, const char *definition); PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); - // For use by projinfo bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index 3b6e366f..24da4dde 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -111,10 +111,19 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct hgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + PJ *TRANSFORMATION(hgridshift,0) { auto Q = new hgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; P->fwd4d = forward_4d; P->inv4d = reverse_4d; diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index f35832e1..3e7a015e 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -142,10 +142,20 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct vgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + + PJ *TRANSFORMATION(vgridshift,0) { auto Q = new vgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; if (!pj_param(P->ctx, P->params, "tgrids").i) { proj_log_error(P, "vgridshift: +grids parameter missing."); diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index f23c2588..734ffc5d 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -270,10 +270,20 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct xyzgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + + PJ *TRANSFORMATION(xyzgridshift,0) { auto Q = new xyzgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; P->fwd4d = nullptr; P->inv4d = nullptr; diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp index 89d66797..5cd32f68 100644 --- a/test/unit/test_network.cpp +++ b/test/unit/test_network.cpp @@ -749,6 +749,8 @@ TEST(networking, curl_hgridshift) { c.xyz.z = 0; c = proj_trans(P, PJ_FWD, c); + proj_assign_context(P, ctx); // (dummy) test context reassignment + proj_destroy(P); proj_context_destroy(ctx); @@ -778,6 +780,8 @@ TEST(networking, curl_vgridshift) { c.xyz.z = 0; c = proj_trans(P, PJ_FWD, c); + proj_assign_context(P, ctx); // (dummy) test context reassignment + proj_destroy(P); proj_context_destroy(ctx); -- cgit v1.2.3 From be596500398778c6d0fc31c0c15aa8b9df7aa3db Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 12:34:24 +0100 Subject: xyzgridshift: implement defered grid loading in network context --- src/transformations/xyzgridshift.cpp | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp index 734ffc5d..a76f3255 100644 --- a/src/transformations/xyzgridshift.cpp +++ b/src/transformations/xyzgridshift.cpp @@ -46,6 +46,7 @@ struct xyzgridshiftData { PJ *cart = nullptr; bool grid_ref_is_input = true; ListOfGenericGrids grids{}; + bool defer_grid_opening = false; double multiplier = 1.0; }; } // anonymous namespace @@ -68,12 +69,20 @@ static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, // --------------------------------------------------------------------------- static bool get_grid_values(PJ* P, - const xyzgridshiftData* Q, + xyzgridshiftData* Q, const PJ_LP& lp, double& dx, double& dy, double& dz) { + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_generic_grid_init(P, "grids"); + if ( proj_errno(P) ) { + return false; + } + } + auto grid = findGrid(Q->grids, lp); if( !grid ) { return false; @@ -169,7 +178,7 @@ static bool get_grid_values(PJ* P, // --------------------------------------------------------------------------- static PJ_COORD iterative_adjustment(PJ* P, - const xyzgridshiftData* Q, + xyzgridshiftData* Q, const PJ_COORD& pointInit, double factor) { @@ -204,7 +213,7 @@ static PJ_COORD iterative_adjustment(PJ* P, // --------------------------------------------------------------------------- static PJ_COORD direct_adjustment(PJ* P, - const xyzgridshiftData* Q, + xyzgridshiftData* Q, PJ_COORD point, double factor) { @@ -328,11 +337,16 @@ PJ *TRANSFORMATION(xyzgridshift,0) { Q->multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; } - Q->grids = proj_generic_grid_init(P, "grids"); - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "xyzgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + Q->grids = proj_generic_grid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "xyzgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } return P; -- cgit v1.2.3 From f934c002c0742dc4bb6fcda1ff44e4035f472ce8 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 17:41:33 +0100 Subject: CurlFileHandle: code improvement. No functional change --- src/filemanager.cpp | 96 +++++++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 43 deletions(-) diff --git a/src/filemanager.cpp b/src/filemanager.cpp index dabb46e0..a279a6cf 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -542,20 +542,52 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename) { #ifdef CURL_ENABLED struct CurlFileHandle { - CURL *m_handle = nullptr; - std::string m_headers; + std::string m_url; + CURL *m_handle; + std::string m_headers{}; std::string m_lastval{}; + char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; CurlFileHandle(const CurlFileHandle &) = delete; CurlFileHandle &operator=(const CurlFileHandle &) = delete; - explicit CurlFileHandle(CURL *handle, std::string &&headers) - : m_handle(handle), m_headers(std::move(headers)) {} + explicit CurlFileHandle(const char *url, CURL *handle); ~CurlFileHandle(); + + static PROJ_NETWORK_HANDLE * + open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *); }; // --------------------------------------------------------------------------- +CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) + : m_url(url), m_handle(handle) { + curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(handle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); +} + +// --------------------------------------------------------------------------- + CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } // --------------------------------------------------------------------------- @@ -575,31 +607,18 @@ static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, // --------------------------------------------------------------------------- -static PROJ_NETWORK_HANDLE * -pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, - size_t size_to_read, void *buffer, size_t *out_size_read, - size_t error_string_max_size, char *out_error_string, void *) { +PROJ_NETWORK_HANDLE *CurlFileHandle::open(PJ_CONTEXT *, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *) { CURL *hCurlHandle = curl_easy_init(); if (!hCurlHandle) return nullptr; - curl_easy_setopt(hCurlHandle, CURLOPT_URL, url); - - if (getenv("PROJ_CURL_VERBOSE")) - curl_easy_setopt(hCurlHandle, CURLOPT_VERBOSE, 1); -// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. -#if LIBCURL_VERSION_NUM >= 0x073600 - curl_easy_setopt(hCurlHandle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); -#endif - - // Enable following redirections. Requires libcurl 7.10.1 at least. - curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(hCurlHandle, CURLOPT_MAXREDIRS, 10); - - if (getenv("PROJ_UNSAFE_SSL")) { - curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(hCurlHandle, CURLOPT_SSL_VERIFYHOST, 0L); - } + auto file = + std::unique_ptr(new CurlFileHandle(url, hCurlHandle)); char szBuffer[128]; sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, @@ -616,9 +635,7 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); - char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; - szCurlErrBuf[0] = '\0'; - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + file->m_szCurlErrBuf[0] = '\0'; curl_easy_perform(hCurlHandle); @@ -631,19 +648,16 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); - if (response_code == 0 || response_code >= 300) { if (out_error_string) { - if (szCurlErrBuf[0]) { + if (file->m_szCurlErrBuf[0]) { snprintf(out_error_string, error_string_max_size, "%s", - szCurlErrBuf); + file->m_szCurlErrBuf); } else { snprintf(out_error_string, error_string_max_size, "HTTP error %ld: %s", response_code, body.c_str()); } } - curl_easy_cleanup(hCurlHandle); return nullptr; } if (out_error_string && error_string_max_size) { @@ -655,8 +669,8 @@ pj_curl_open(PJ_CONTEXT *, const char *url, unsigned long long offset, } *out_size_read = std::min(size_to_read, body.size()); - return reinterpret_cast( - new CurlFileHandle(hCurlHandle, std::move(headers))); + file->m_headers = std::move(headers); + return reinterpret_cast(file.release()); } // --------------------------------------------------------------------------- @@ -685,9 +699,7 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); - char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; - szCurlErrBuf[0] = '\0'; - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf); + handle->m_szCurlErrBuf[0] = '\0'; curl_easy_perform(hCurlHandle); @@ -697,13 +709,11 @@ static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); - curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, nullptr); - if (response_code == 0 || response_code >= 300) { if (out_error_string) { - if (szCurlErrBuf[0]) { + if (handle->m_szCurlErrBuf[0]) { snprintf(out_error_string, error_string_max_size, "%s", - szCurlErrBuf); + handle->m_szCurlErrBuf); } else { snprintf(out_error_string, error_string_max_size, "HTTP error %ld: %s", response_code, body.c_str()); @@ -772,7 +782,7 @@ static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { #ifdef CURL_ENABLED - ctx->networking.open = pj_curl_open; + ctx->networking.open = CurlFileHandle::open; ctx->networking.close = pj_curl_close; ctx->networking.read_range = pj_curl_read_range; ctx->networking.get_header_value = pj_curl_get_header_value; -- cgit v1.2.3 From 28e1770f27bb335d29bfa44a5c963904007b5e73 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Dec 2019 17:42:04 +0100 Subject: CurlFileHandle: set UserAgent --- src/filemanager.cpp | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/filemanager.cpp b/src/filemanager.cpp index a279a6cf..625dca03 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -61,8 +61,22 @@ class MyMutex { #include // for sqlite3_snprintf #endif +#if defined(__linux) +#include +#elif defined(_WIN32) +#include +#elif defined(__MACH__) && defined(__APPLE__) +#include +#elif defined(__FreeBSD__) +#include +#include +#endif + //! @cond Doxygen_Suppress +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + using namespace NS_PROJ::internal; NS_PROJ_START @@ -546,6 +560,7 @@ struct CurlFileHandle { CURL *m_handle; std::string m_headers{}; std::string m_lastval{}; + std::string m_useragent{}; char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; CurlFileHandle(const CurlFileHandle &) = delete; @@ -562,6 +577,67 @@ struct CurlFileHandle { // --------------------------------------------------------------------------- +static std::string GetExecutableName() { +#if defined(__linux) + std::string path; + path.resize(1024); + const auto ret = readlink("/proc/self/exe", &path[0], path.size()); + if (ret > 0) { + path.resize(ret); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(_WIN32) + std::string path; + path.resize(1024); + if (GetModuleFileNameA(nullptr, &path[0], + static_cast(path.size()))) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('\\'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__MACH__) && defined(__APPLE__) + std::string path; + path.resize(1024); + uint32_t size = static_cast(path.size()); + if (_NSGetExecutablePath(&path[0], &size) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__FreeBSD__) + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PATHNAME; + mib[3] = -1; + std::string path; + path.resize(1024); + size_t size = path.size(); + if (sysctl(mib, 4, &path[0], &size, nullptr, 0) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#endif + + return std::string(); +} + +// --------------------------------------------------------------------------- + CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) : m_url(url), m_handle(handle) { curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); @@ -584,6 +660,16 @@ CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) } curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); + + if (getenv("PROJ_NO_USERAGENT") == nullptr) { + m_useragent = "PROJ " STR(PROJ_VERSION_MAJOR) "." STR( + PROJ_VERSION_MINOR) "." STR(PROJ_VERSION_PATCH); + const auto exeName = GetExecutableName(); + if (!exeName.empty()) { + m_useragent = exeName + " using " + m_useragent; + } + curl_easy_setopt(handle, CURLOPT_USERAGENT, m_useragent.data()); + } } // --------------------------------------------------------------------------- -- cgit v1.2.3 From 830f94a8117eff270acd2ef928850a9de5e164a9 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sun, 29 Dec 2019 23:57:23 +0100 Subject: proj_hgrid_value(): do not apply compensation for west-oriented longitude_offset. Fixes regression due to grid refactoring work on proj=deformation use case --- src/apply_gridshift.cpp | 16 ++++++++-------- src/grids.cpp | 38 ++++++++++++++++++++++++-------------- src/grids.hpp | 4 ++-- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp index b994474b..4ef86fc0 100644 --- a/src/apply_gridshift.cpp +++ b/src/apply_gridshift.cpp @@ -119,7 +119,7 @@ ListOfHGrids proj_hgrid_init(PJ* P, const char *gridkey) { typedef struct { pj_int32 lam, phi; } ILP; -static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { +static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid, bool compensateNTConvention) { PJ_LP val, frct; ILP indx; int in; @@ -164,10 +164,10 @@ static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { float f10Lon = 0, f10Lat = 0; float f01Lon = 0, f01Lat = 0; float f11Lon = 0, f11Lat = 0; - if( !grid->valueAt(indx.lam, indx.phi, f00Lon, f00Lat) || - !grid->valueAt(indx.lam + 1, indx.phi, f10Lon, f10Lat) || - !grid->valueAt(indx.lam, indx.phi + 1, f01Lon, f01Lat) || - !grid->valueAt(indx.lam + 1, indx.phi + 1, f11Lon, f11Lat) ) + if( !grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, f00Lat) || + !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, f10Lat) || + !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, f01Lat) || + !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, f11Lon, f11Lat) ) { return val; } @@ -210,7 +210,7 @@ PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* gri tb.lam = adjlon (tb.lam - M_PI) + M_PI; - t = nad_intr (tb, grid); + t = nad_intr (tb, grid, true); if (t.lam == HUGE_VAL) return t; @@ -224,7 +224,7 @@ PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* gri t.phi = tb.phi - t.phi; do { - del = nad_intr(t, grid); + del = nad_intr(t, grid, true); /* We can possibly go outside of the initial guessed grid, so try */ /* to fetch a new grid into which iterate... */ @@ -297,7 +297,7 @@ PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids& grids, PJ_LP lp) { lp.lam = adjlon(lp.lam - M_PI) + M_PI; - out = nad_intr(lp, grid); + out = nad_intr(lp, grid, false); if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); diff --git a/src/grids.cpp b/src/grids.cpp index a3d984de..5a99106b 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -1431,14 +1431,15 @@ class NullHorizontalShiftGrid : public HorizontalShiftGrid { bool isNullGrid() const override { return true; } - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- -bool NullHorizontalShiftGrid::valueAt(int, int, float &lonShift, +bool NullHorizontalShiftGrid::valueAt(int, int, bool, float &lonShift, float &latShift) const { lonShift = 0.0f; latShift = 0.0f; @@ -1471,7 +1472,8 @@ class NTv1Grid : public HorizontalShiftGrid { ~NTv1Grid() override; - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); @@ -1549,7 +1551,8 @@ NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, // --------------------------------------------------------------------------- -bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool NTv1Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); double two_doubles[2]; @@ -1566,7 +1569,8 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { /* convert seconds to radians */ latShift = static_cast(two_doubles[0] * ((M_PI / 180.0) / 3600.0)); // west longitude positive convention ! - lonShift = -static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); return true; } @@ -1589,7 +1593,8 @@ class CTable2Grid : public HorizontalShiftGrid { ~CTable2Grid() override; - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr fp, const std::string &filename); @@ -1659,8 +1664,8 @@ CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, std::unique_ptr fp, // --------------------------------------------------------------------------- -bool CTable2Grid::valueAt(int x, int y, float &lonShift, - float &latShift) const { +bool CTable2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_floats[2]; @@ -1675,7 +1680,7 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, latShift = two_floats[1]; // west longitude positive convention ! - lonShift = -two_floats[0]; + lonShift = (compensateNTConvention ? -1 : 1) * two_floats[0]; return true; } @@ -1725,7 +1730,8 @@ class NTv2Grid : public HorizontalShiftGrid { m_name(nameIn), m_ctx(ctx), m_fp(fp), m_offset(offsetIn), m_mustSwap(mustSwapIn) {} - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; @@ -1735,7 +1741,8 @@ class NTv2Grid : public HorizontalShiftGrid { // --------------------------------------------------------------------------- -bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool NTv2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_float[2]; @@ -1755,7 +1762,8 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { /* convert seconds to radians */ latShift = static_cast(two_float[0] * ((M_PI / 180.0) / 3600.0)); // west longitude positive convention ! - lonShift = -static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast(two_float[1] * ((M_PI / 180.0) / 3600.0)); return true; } @@ -1938,7 +1946,8 @@ class GTiffHGrid : public HorizontalShiftGrid { ~GTiffHGrid() override; - bool valueAt(int x, int y, float &lonShift, float &latShift) const override; + bool valueAt(int x, int y, bool, float &lonShift, + float &latShift) const override; void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr &&subgrid); @@ -1968,7 +1977,8 @@ GTiffHGrid::~GTiffHGrid() = default; // --------------------------------------------------------------------------- -bool GTiffHGrid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool GTiffHGrid::valueAt(int x, int y, bool, float &lonShift, + float &latShift) const { if (!m_grid->valueAt(m_idxLatShift, x, y, latShift) || !m_grid->valueAt(m_idxLonShift, x, y, lonShift)) { return false; diff --git a/src/grids.hpp b/src/grids.hpp index fde3eb3e..aa852ef6 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -132,8 +132,8 @@ class HorizontalShiftGrid : public Grid { const HorizontalShiftGrid *gridAt(double lon, double lat) const; // x = 0 is western-most column, y = 0 is southern-most line - virtual bool valueAt(int x, int y, float &lonShift, - float &latShift) const = 0; + virtual bool valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const = 0; virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; -- cgit v1.2.3 From 17864e68dc7b34bb730bdc191117e1bd1d5d18ef Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Sat, 28 Dec 2019 23:30:46 +0100 Subject: Travis: fix OSX builds --- travis/osx/before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/osx/before_install.sh b/travis/osx/before_install.sh index 4a52cd30..e457ec8c 100755 --- a/travis/osx/before_install.sh +++ b/travis/osx/before_install.sh @@ -10,7 +10,7 @@ brew install ccache #brew upgrade libtiff brew install doxygen #brew install md5sha1sum -brew reinstall python +#brew reinstall python brew reinstall wget ./travis/before_install_pip.sh -- cgit v1.2.3