diff options
102 files changed, 5858 insertions, 395 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f1ff7a2e..4f140e72 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,7 +16,7 @@ task: folder: $HOME/.ccache pkginstall_script: - - pkg install -y autoconf automake libtool pkgconf sqlite3 wget unzip ccache gmake + - pkg install -y autoconf automake libtool pkgconf sqlite3 tiff wget unzip ccache gmake download_grid_script: - wget https://download.osgeo.org/proj/proj-datumgrid-1.8.zip - (cd data && unzip -o ../proj-datumgrid-1.8.zip) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5715389..2125b93a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,9 @@ elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") set(PROJ_CXX_WARN_FLAGS -Wall) endif() endif() + +add_definitions(-DNOMINMAX) + set(PROJ_C_WARN_FLAGS "${PROJ_C_WARN_FLAGS}" CACHE STRING "C flags used to compile PROJ targets") set(PROJ_CXX_WARN_FLAGS "${PROJ_CXX_WARN_FLAGS}" @@ -134,6 +137,35 @@ IF("${SQLITE3_VERSION}" VERSION_LESS "3.11") ENDIF() ################################################################################ +# Check for libtiff +################################################################################ + +option(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED "Disable TIFF support (strongly discouraged !)" OFF) +mark_as_advanced(DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) +if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) + find_package(TIFF REQUIRED) + if(NOT TIFF_FOUND) + message(SEND_ERROR "libtiff dependency not found!") + endif() + add_definitions(-DTIFF_ENABLED) +endif() + +################################################################################ +# Check for curl +################################################################################ + +option(ENABLE_CURL "Enable Curl support" ON) +if(ENABLE_CURL) + find_package(CURL REQUIRED) + if(CURL_FOUND) + boost_report_value(CURL_FOUND) + else() + message(SEND_ERROR "curl dependency not found!") + endif() + add_definitions(-DCURL_ENABLED) +endif() + +################################################################################ # threading configuration ################################################################################ set(CMAKE_THREAD_PREFER_PTHREAD TRUE) @@ -772,7 +772,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox +INPUT = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox src/filemanager.cpp # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/appveyor.yml b/appveyor.yml index a631c3e8..2511854f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,9 +27,10 @@ build_script: - set PATH=%CD%;%PATH% - cd .. - vcpkg install sqlite3:"%platform%"-windows + - vcpkg install tiff:"%platform%"-windows + - vcpkg install curl:"%platform%"-windows - set SQLITE3_BIN=%APPVEYOR_BUILD_FOLDER%\sqlite3\bin - mkdir %SQLITE3_BIN% - - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\sqlite3.dll %SQLITE3_BIN% - ps: | appveyor DownloadFile https://sqlite.org/2018/sqlite-tools-win32-x86-3250100.zip 7z x sqlite-tools-win32-x86-3250100.zip @@ -47,6 +48,8 @@ build_script: - set PROJ_DIR=%APPVEYOR_BUILD_FOLDER%\proj_dir - cmake -G "%VS_FULL%" .. -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBPROJ_SHARED="%BUILD_LIBPROJ_SHARED%" -DCMAKE_C_FLAGS="/WX" -DCMAKE_CXX_FLAGS="/WX" -DCMAKE_TOOLCHAIN_FILE=C:/projects/proj/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX="%PROJ_DIR%" - cmake --build . --config Release --target install + - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\*.dll %PROJ_DIR%\bin + - dir %PROJ_DIR%\bin test_script: - echo test_script diff --git a/cmake/ProjTest.cmake b/cmake/ProjTest.cmake index ad64d0ba..6abfbcd7 100644 --- a/cmake/ProjTest.cmake +++ b/cmake/ProjTest.cmake @@ -26,8 +26,13 @@ function(proj_add_test_script_sh SH_NAME BIN_USE) COMMAND ${PROJECT_SOURCE_DIR}/test/cli/${SH_NAME} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${BIN_USE}} ) - set_tests_properties( ${testname} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + if(MSVC) + set_tests_properties( ${testname} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + else() + set_tests_properties( ${testname} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + endif() endif() endif() @@ -43,8 +48,14 @@ function(proj_add_gie_test TESTNAME TESTCASE) COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${GIE_BIN} ${TESTFILE} ) - set_tests_properties( ${TESTNAME} - PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + + if(MSVC) + set_tests_properties( ${TESTNAME} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data") + else() + set_tests_properties( ${TESTNAME} + PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data") + endif() endfunction() diff --git a/configure.ac b/configure.ac index 0ea8e04a..afde2712 100644 --- a/configure.ac +++ b/configure.ac @@ -20,6 +20,8 @@ AC_PROG_LN_S AC_PROG_MAKE_SET AM_PROG_LIBTOOL +PKG_PROG_PKG_CONFIG + dnl Enable as much warnings as possible AX_CFLAGS_WARN_ALL(C_WFLAGS) AX_CXXFLAGS_WARN_ALL(CXX_WFLAGS) @@ -166,8 +168,8 @@ AC_SUBST(C_WFLAGS,$C_WFLAGS) AC_SUBST(CXX_WFLAGS,$CXX_WFLAGS) AC_SUBST(NO_ZERO_AS_NULL_POINTER_CONSTANT_FLAG,$NO_ZERO_AS_NULL_POINTER_CONSTANT_FLAG) -CFLAGS="${CFLAGS} -fvisibility=hidden" -CXXFLAGS="${CXXFLAGS} -fvisibility=hidden" +CFLAGS="${CFLAGS} -fvisibility=hidden -DNOMINMAX" +CXXFLAGS="${CXXFLAGS} -fvisibility=hidden -DNOMINMAX" dnl Checks for libraries. save_CFLAGS="$CFLAGS" @@ -295,6 +297,78 @@ if test x"$SQLITE3_CHECK" != x"yes" ; then fi dnl --------------------------------------------------------------------------- +dnl Check for libtiff +dnl --------------------------------------------------------------------------- + +AC_ARG_ENABLE([tiff-is-strongly-discouraged], + AS_HELP_STRING([--disable-tiff-is-strongly-discouraged], + [Disable TIFF support. Strongly discouraged !]), + [enable_tiff=no], + [enable_tiff=yes]) + +if test "x$enable_tiff" = "xyes" -o "x$enable_tiff" = ""; then + if test "x$TIFF_CFLAGS$TIFF_LIBS" = "x" ; then + if $PKG_CONFIG libtiff; then + PKG_CHECK_MODULES([TIFF], [libtiff]) + else + PKG_CHECK_MODULES([TIFF], [libtiff-4]) + fi + fi + TIFF_ENABLED_FLAGS=-DTIFF_ENABLED +fi +AC_SUBST(TIFF_CFLAGS,$TIFF_CFLAGS) +AC_SUBST(TIFF_LIBS,$TIFF_LIBS) +AC_SUBST(TIFF_ENABLED_FLAGS,$TIFF_ENABLED_FLAGS) + +dnl --------------------------------------------------------------------------- +dnl Check for curl +dnl --------------------------------------------------------------------------- + +FOUND_CURL=no +CURL_CFLAGS= +CURL_LIB= + +AC_ARG_WITH(curl, + [ --with-curl[=ARG] Enable curl support (ARG=path to curl-config.)],,,) + +dnl Clear some cache variables +unset ac_cv_path_LIBCURL + +if test "`basename xx/$with_curl`" = "curl-config" ; then + LIBCURL_CONFIG="$with_curl" +elif test "$with_curl" = "no" ; then + LIBCURL_CONFIG=no +else + AC_PATH_PROG(LIBCURL_CONFIG, curl-config, not-found) +fi + +if test "$LIBCURL_CONFIG" = "not-found" ; then + AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl]) +elif test "$LIBCURL_CONFIG" != "no" ; then + + CURL_VERNUM=`$LIBCURL_CONFIG --vernum` + CURL_VER=`$LIBCURL_CONFIG --version | awk '{print $2}'` + + AC_MSG_RESULT([ found libcurl version $CURL_VER]) + + AC_CHECK_LIB(curl,curl_global_init,FOUND_CURL=yes,FOUND_CURL=no,`$LIBCURL_CONFIG --libs`) + + if test "$FOUND_CURL" = "no" ; then + AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl]) + fi + CURL_ENABLED_FLAGS=-DCURL_ENABLED +fi + +if test "$FOUND_CURL" = "yes" ; then + CURL_CFLAGS=`$LIBCURL_CONFIG --cflags` + CURL_LIBS=`$LIBCURL_CONFIG --libs` +fi + +AC_SUBST(CURL_CFLAGS,$CURL_CFLAGS) +AC_SUBST(CURL_LIBS,$CURL_LIBS) +AC_SUBST(CURL_ENABLED_FLAGS,$CURL_ENABLED_FLAGS) + +dnl --------------------------------------------------------------------------- dnl Check for external Google Test dnl --------------------------------------------------------------------------- diff --git a/data/Makefile.am b/data/Makefile.am index e7c38d7c..09fa1989 100644 --- a/data/Makefile.am +++ b/data/Makefile.am @@ -1,6 +1,6 @@ DATAPATH = $(top_srcdir)/data -pkgdata_DATA = GL27 nad.lst nad27 nad83 world other.extra \ +pkgdata_DATA = proj.ini GL27 nad.lst nad27 nad83 world other.extra \ CH null \ ITRF2000 ITRF2008 ITRF2014 proj.db \ projjson.schema.json @@ -38,12 +38,51 @@ SQL_ORDERED_LIST = sql/begin.sql \ sql/customizations.sql \ sql/commit.sql -EXTRA_DIST = GL27 nad.lst nad27 nad83 \ +EXTRA_DIST = proj.ini GL27 nad.lst nad27 nad83 \ world other.extra \ CH \ ITRF2000 ITRF2008 ITRF2014 \ projjson.schema.json \ - CMakeLists.txt tests/test_nodata.gtx null \ + CMakeLists.txt \ + tests/test_nodata.gtx \ + tests/test_vgrid_bigendian_bigtiff.tif \ + tests/test_vgrid_bigendian.tif \ + tests/test_vgrid_bigtiff.tif \ + tests/test_vgrid_bottomup_with_matrix.tif \ + tests/test_vgrid_bottomup_with_scale.tif \ + tests/test_vgrid_deflate_floatingpointpredictor.tif \ + tests/test_vgrid_deflate.tif \ + tests/test_vgrid_float64.tif \ + tests/test_vgrid_in_second_channel.tif \ + tests/test_vgrid_int16.tif \ + tests/test_vgrid_int32.tif \ + tests/test_vgrid_uint32.tif \ + tests/test_vgrid_invalid_channel_type.tif \ + tests/test_vgrid_nodata.tif \ + tests/test_vgrid_pixelisarea.tif \ + tests/test_vgrid_pixelispoint.tif \ + tests/test_vgrid_uint16.tif \ + tests/test_vgrid_uint16_with_scale_offset.tif \ + tests/test_vgrid_unsupported_byte.tif \ + tests/test_vgrid_with_overview.tif \ + tests/test_vgrid_with_subgrid.tif \ + tests/test_hgrid.tif \ + tests/test_hgrid_separate.tif \ + tests/test_hgrid_tiled.tif \ + tests/test_hgrid_tiled_separate.tif \ + tests/test_hgrid_strip.tif \ + tests/test_hgrid_positive_west.tif \ + tests/test_hgrid_lon_shift_first.tif \ + tests/test_hgrid_radian.tif \ + tests/test_hgrid_degree.tif \ + tests/test_hgrid_with_overview.tif \ + tests/test_hgrid_extra_ifd_with_other_info.tif \ + tests/test_hgrid_with_subgrid.tif \ + tests/test_hgrid_with_subgrid_no_grid_name.tif \ + tests/subset_of_gr3df97a.tif \ + tests/egm96_15_uncompressed_truncated.tif \ + tests/test_vgrid_single_strip_truncated.tif \ + null \ generate_all_sql_in.cmake sql_filelist.cmake \ $(SQL_ORDERED_LIST) diff --git a/data/proj.ini b/data/proj.ini new file mode 100644 index 00000000..f42bc940 --- /dev/null +++ b/data/proj.ini @@ -0,0 +1,10 @@ +[general] +; Lines starting by ; are commented lines. +; + +; Network capabilities disabled by default. +; Can be overriden with the PROJ_NETWORK=ON environment variable. +; network = on + +; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. +cdn_endpoint = https://cdn.proj.org diff --git a/data/tests/egm96_15_uncompressed_truncated.tif b/data/tests/egm96_15_uncompressed_truncated.tif Binary files differnew file mode 100644 index 00000000..bd34a7e2 --- /dev/null +++ b/data/tests/egm96_15_uncompressed_truncated.tif diff --git a/data/tests/subset_of_gr3df97a.tif b/data/tests/subset_of_gr3df97a.tif Binary files differnew file mode 100644 index 00000000..a98783f3 --- /dev/null +++ b/data/tests/subset_of_gr3df97a.tif diff --git a/data/tests/test_hgrid.tif b/data/tests/test_hgrid.tif Binary files differnew file mode 100644 index 00000000..94718c21 --- /dev/null +++ b/data/tests/test_hgrid.tif diff --git a/data/tests/test_hgrid_degree.tif b/data/tests/test_hgrid_degree.tif Binary files differnew file mode 100644 index 00000000..d06782ec --- /dev/null +++ b/data/tests/test_hgrid_degree.tif diff --git a/data/tests/test_hgrid_extra_ifd_with_other_info.tif b/data/tests/test_hgrid_extra_ifd_with_other_info.tif Binary files differnew file mode 100644 index 00000000..b7e67af7 --- /dev/null +++ b/data/tests/test_hgrid_extra_ifd_with_other_info.tif diff --git a/data/tests/test_hgrid_lon_shift_first.tif b/data/tests/test_hgrid_lon_shift_first.tif Binary files differnew file mode 100644 index 00000000..395f0743 --- /dev/null +++ b/data/tests/test_hgrid_lon_shift_first.tif diff --git a/data/tests/test_hgrid_positive_west.tif b/data/tests/test_hgrid_positive_west.tif Binary files differnew file mode 100644 index 00000000..4ebc17cc --- /dev/null +++ b/data/tests/test_hgrid_positive_west.tif diff --git a/data/tests/test_hgrid_radian.tif b/data/tests/test_hgrid_radian.tif Binary files differnew file mode 100644 index 00000000..30219ccd --- /dev/null +++ b/data/tests/test_hgrid_radian.tif diff --git a/data/tests/test_hgrid_separate.tif b/data/tests/test_hgrid_separate.tif Binary files differnew file mode 100644 index 00000000..ef2ca575 --- /dev/null +++ b/data/tests/test_hgrid_separate.tif diff --git a/data/tests/test_hgrid_strip.tif b/data/tests/test_hgrid_strip.tif Binary files differnew file mode 100644 index 00000000..e38fc609 --- /dev/null +++ b/data/tests/test_hgrid_strip.tif diff --git a/data/tests/test_hgrid_tiled.tif b/data/tests/test_hgrid_tiled.tif Binary files differnew file mode 100644 index 00000000..b0d5dd8b --- /dev/null +++ b/data/tests/test_hgrid_tiled.tif diff --git a/data/tests/test_hgrid_tiled_separate.tif b/data/tests/test_hgrid_tiled_separate.tif Binary files differnew file mode 100644 index 00000000..d7e0934f --- /dev/null +++ b/data/tests/test_hgrid_tiled_separate.tif diff --git a/data/tests/test_hgrid_with_overview.tif b/data/tests/test_hgrid_with_overview.tif Binary files differnew file mode 100644 index 00000000..d7453b49 --- /dev/null +++ b/data/tests/test_hgrid_with_overview.tif diff --git a/data/tests/test_hgrid_with_subgrid.tif b/data/tests/test_hgrid_with_subgrid.tif Binary files differnew file mode 100644 index 00000000..46a8f2f4 --- /dev/null +++ b/data/tests/test_hgrid_with_subgrid.tif diff --git a/data/tests/test_hgrid_with_subgrid_no_grid_name.tif b/data/tests/test_hgrid_with_subgrid_no_grid_name.tif Binary files differnew file mode 100644 index 00000000..974699b5 --- /dev/null +++ b/data/tests/test_hgrid_with_subgrid_no_grid_name.tif diff --git a/data/tests/test_vgrid_bigendian.tif b/data/tests/test_vgrid_bigendian.tif Binary files differnew file mode 100644 index 00000000..5cf4a039 --- /dev/null +++ b/data/tests/test_vgrid_bigendian.tif diff --git a/data/tests/test_vgrid_bigendian_bigtiff.tif b/data/tests/test_vgrid_bigendian_bigtiff.tif Binary files differnew file mode 100644 index 00000000..a586b85f --- /dev/null +++ b/data/tests/test_vgrid_bigendian_bigtiff.tif diff --git a/data/tests/test_vgrid_bigtiff.tif b/data/tests/test_vgrid_bigtiff.tif Binary files differnew file mode 100644 index 00000000..2a01893a --- /dev/null +++ b/data/tests/test_vgrid_bigtiff.tif diff --git a/data/tests/test_vgrid_bottomup_with_matrix.tif b/data/tests/test_vgrid_bottomup_with_matrix.tif Binary files differnew file mode 100644 index 00000000..90f637dc --- /dev/null +++ b/data/tests/test_vgrid_bottomup_with_matrix.tif diff --git a/data/tests/test_vgrid_bottomup_with_scale.tif b/data/tests/test_vgrid_bottomup_with_scale.tif Binary files differnew file mode 100644 index 00000000..636b7dc7 --- /dev/null +++ b/data/tests/test_vgrid_bottomup_with_scale.tif diff --git a/data/tests/test_vgrid_deflate.tif b/data/tests/test_vgrid_deflate.tif Binary files differnew file mode 100644 index 00000000..ee3b5f0a --- /dev/null +++ b/data/tests/test_vgrid_deflate.tif diff --git a/data/tests/test_vgrid_deflate_floatingpointpredictor.tif b/data/tests/test_vgrid_deflate_floatingpointpredictor.tif Binary files differnew file mode 100644 index 00000000..5fd7b9fa --- /dev/null +++ b/data/tests/test_vgrid_deflate_floatingpointpredictor.tif diff --git a/data/tests/test_vgrid_float64.tif b/data/tests/test_vgrid_float64.tif Binary files differnew file mode 100644 index 00000000..16b3e790 --- /dev/null +++ b/data/tests/test_vgrid_float64.tif diff --git a/data/tests/test_vgrid_in_second_channel.tif b/data/tests/test_vgrid_in_second_channel.tif Binary files differnew file mode 100644 index 00000000..d377f8b7 --- /dev/null +++ b/data/tests/test_vgrid_in_second_channel.tif diff --git a/data/tests/test_vgrid_int16.tif b/data/tests/test_vgrid_int16.tif Binary files differnew file mode 100644 index 00000000..1c69b5d6 --- /dev/null +++ b/data/tests/test_vgrid_int16.tif diff --git a/data/tests/test_vgrid_int32.tif b/data/tests/test_vgrid_int32.tif Binary files differnew file mode 100644 index 00000000..1b6dfd7b --- /dev/null +++ b/data/tests/test_vgrid_int32.tif diff --git a/data/tests/test_vgrid_invalid_channel_type.tif b/data/tests/test_vgrid_invalid_channel_type.tif Binary files differnew file mode 100644 index 00000000..ec9e641f --- /dev/null +++ b/data/tests/test_vgrid_invalid_channel_type.tif diff --git a/data/tests/test_vgrid_nodata.tif b/data/tests/test_vgrid_nodata.tif Binary files differnew file mode 100644 index 00000000..65ec5343 --- /dev/null +++ b/data/tests/test_vgrid_nodata.tif diff --git a/data/tests/test_vgrid_pixelisarea.tif b/data/tests/test_vgrid_pixelisarea.tif Binary files differnew file mode 100644 index 00000000..a5409f66 --- /dev/null +++ b/data/tests/test_vgrid_pixelisarea.tif diff --git a/data/tests/test_vgrid_pixelispoint.tif b/data/tests/test_vgrid_pixelispoint.tif Binary files differnew file mode 100644 index 00000000..cfeb598f --- /dev/null +++ b/data/tests/test_vgrid_pixelispoint.tif diff --git a/data/tests/test_vgrid_single_strip_truncated.tif b/data/tests/test_vgrid_single_strip_truncated.tif Binary files differnew file mode 100644 index 00000000..9a0030f6 --- /dev/null +++ b/data/tests/test_vgrid_single_strip_truncated.tif diff --git a/data/tests/test_vgrid_uint16.tif b/data/tests/test_vgrid_uint16.tif Binary files differnew file mode 100644 index 00000000..a03d9a73 --- /dev/null +++ b/data/tests/test_vgrid_uint16.tif diff --git a/data/tests/test_vgrid_uint16_with_scale_offset.tif b/data/tests/test_vgrid_uint16_with_scale_offset.tif Binary files differnew file mode 100644 index 00000000..b08fa4a3 --- /dev/null +++ b/data/tests/test_vgrid_uint16_with_scale_offset.tif diff --git a/data/tests/test_vgrid_uint32.tif b/data/tests/test_vgrid_uint32.tif Binary files differnew file mode 100644 index 00000000..cae7e9e7 --- /dev/null +++ b/data/tests/test_vgrid_uint32.tif diff --git a/data/tests/test_vgrid_unsupported_byte.tif b/data/tests/test_vgrid_unsupported_byte.tif Binary files differnew file mode 100644 index 00000000..ccf03fc8 --- /dev/null +++ b/data/tests/test_vgrid_unsupported_byte.tif diff --git a/data/tests/test_vgrid_with_overview.tif b/data/tests/test_vgrid_with_overview.tif Binary files differnew file mode 100644 index 00000000..aa15aa1d --- /dev/null +++ b/data/tests/test_vgrid_with_overview.tif diff --git a/data/tests/test_vgrid_with_subgrid.tif b/data/tests/test_vgrid_with_subgrid.tif Binary files differnew file mode 100644 index 00000000..5c7584c4 --- /dev/null +++ b/data/tests/test_vgrid_with_subgrid.tif diff --git a/docs/source/apps/cs2cs.rst b/docs/source/apps/cs2cs.rst index d7f0e9ad..a4069472 100644 --- a/docs/source/apps/cs2cs.rst +++ b/docs/source/apps/cs2cs.rst @@ -175,6 +175,14 @@ normally be in DMS format (use ``-f %.12f`` for decimal degrees with 12 decimal places), while projected (cartesian) coordinates will be in linear (meter, feet) units. +Use of remote grids +------------------- + +.. versionadded:: 7.0.0 + +If the :envvar:`PROJ_NETWORK` environment variable is set to ``ON``, +:program:`cs2cs` will attempt to use remote grids stored on CDN (Content +Delivery Network) storage, when they are not available locally. Examples ******** diff --git a/docs/source/apps/projinfo.rst b/docs/source/apps/projinfo.rst index 6236056d..a9b6aecc 100644 --- a/docs/source/apps/projinfo.rst +++ b/docs/source/apps/projinfo.rst @@ -20,7 +20,7 @@ Synopsis | [[--area name_or_code] | [--bbox west_long,south_lat,east_long,north_lat]] | [--spatial-test contains|intersects] | [--crs-extent-use none|both|intersection|smallest] - | [--grid-check none|discard_missing|sort] [--show-superseded] + | [--grid-check none|discard_missing|sort|known_available] [--show-superseded] | [--pivot-crs always|if_no_direct_transformation|never|{auth:code[,auth:code]*}] | [--boundcrs-to-wgs84] | [--main-db-path path] [--aux-db-path path]* @@ -148,12 +148,13 @@ The following control parameters can appear in any order: .. note:: only used for coordinate operation computation -.. option:: --grid-check none|discard_missing|sort +.. option:: --grid-check none|discard_missing|sort|known_available Specify how the presence or absence of a horizontal or vertical shift grid required for a coordinate operation affects the results returned when researching coordinate operations between 2 CRS. - The default strategy is ``sort``: in that case, all candidate + The default strategy is ``sort`` (if :envvar:`PROJ_NETWORK` is not defined). + In that case, all candidate operations are returned, but the actual availability of the grids is used to determine the sorting order. That is, if a coordinate operation involves using a grid that is not available in the PROJ resource directories @@ -163,6 +164,9 @@ The following control parameters can appear in any order: this returns the results as if all the grids where available. The ``discard_missing`` strategy discards results that involve grids not present in the PROJ resource directories. + The ``known_available`` strategy discards results that involve grids not + present in the PROJ resource directories and that are not known of the CDN. + This is the default strategy is :envvar:`PROJ_NETWORK` is set to ``ON``. .. note:: only used for coordinate operation computation diff --git a/docs/source/operations/transformations/index.rst b/docs/source/operations/transformations/index.rst index d4bbf4e0..6bd503d4 100644 --- a/docs/source/operations/transformations/index.rst +++ b/docs/source/operations/transformations/index.rst @@ -19,3 +19,4 @@ systems are based on different datums. molobadekas hgridshift vgridshift + xyzgridshift diff --git a/docs/source/operations/transformations/xyzgridshift.rst b/docs/source/operations/transformations/xyzgridshift.rst new file mode 100644 index 00000000..c49b7592 --- /dev/null +++ b/docs/source/operations/transformations/xyzgridshift.rst @@ -0,0 +1,91 @@ +.. _xyzgridshift: + +================================================================================ +Geocentric grid shift +================================================================================ + +.. versionadded:: 7.0.0 + +Geocentric translation using a grid shift + ++-----------------+-------------------------------------------------------------------+ +| **Domain** | 3D | ++-----------------+-------------------------------------------------------------------+ +| **Input type** | Cartesian coordinates | ++-----------------+-------------------------------------------------------------------+ +| **Output type** | Cartesian coordinates | ++-----------------+-------------------------------------------------------------------+ + +Perform a geocentric translation by bilinear interpolation of dx, dy, dz +translation values from a grid. The grid is referenced against either the +2D geographic CRS corresponding to the input (or sometimes output) CRS. + +This method is described (in French) in :cite:`NTF_88` +and as EPSG operation methode code 9655 in :cite:`IOGP2018` (§2.4.4.1.1 +France geocentric interpolation). + +The translation in the grids are added to the input coordinates in the forward direction, +and subtracted in the reverse direction. +By default (if grid_ref=input_crs), in the forward direction, the input coordinates +are converted to their geographic equivalent to directly read and interpolate from +the grid. In the reverse direction, an iterative method is used to be able to find +the grid locations to read. +If grid_ref=output_crs is used, then the reverse strategy is applied: iterative +method in the forward direction, and direct read in the reverse direction. + +Example +------------------------------------------------------------------------------- + +NTF to RGF93 transformation using gr3df97a.tif grid + +:: + + +proj=pipeline + +step +proj=push +v_3 + +step +proj=cart +ellps=clrk80ign + +step +proj=xyzgridshift +grids=gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 + +step +proj=cart +ellps=GRS80 +inv + +step +proj=pop +v_3 + +Parameters +################################################################################ + +The ellipsoid parameters should be the ones consistent with ``grid_ref``. +They are used to perform a geocentric to geographic conversion to find the +translation parameters. + + +Required ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. include:: ../options/ellps.rst + +.. option:: +grids=<list> + + Comma-separated list of grids to load. If a grid is prefixed by an `@` the + grid is considered optional and PROJ will the not complain if the grid is + not available. + + Grids are expected to be in GeoTIFF format. If no metadata is provided, + the first, second and third samples are assumed to be the geocentric + translation along X, Y and Z axis respectively, in metres. + +Optional ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.. option:: +grid_ref=input_crs/output_crs + + Specify in which CRS the grid is referenced to. The default value is + input_crs. That is the grid is referenced in the geographic CRS corresponding + to the input geocentric CRS. + + If output_crs is specified, the grid is referenced in the geographic CRS corresponding + to the output geocentric CRS. This is for example the case for the French + gr3df97a.tif grid converting from NTF to RGF93, but referenced against RGF93. + Thus in the forward direction (NTF->RGF93), an iterative method is used to find + the appropriate shift. + +.. option:: +multiplier=<value> + + Specify the multiplier to apply to the grid values. Defaults to 1.0 + diff --git a/docs/source/references.bib b/docs/source/references.bib index 0668e10d..8e527ea1 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -230,6 +230,15 @@ Doi = {10.2312/PE/PG/PG2012short/005-010} } +@TechReport{NTF_88, + Title = {Grille de parametres de transformation de coordonnees - GR3DF97A - Notice d'utilisation}, + Author = {IGN}, + Institution = {Service de Geodesie et Nivellement, Institut Geographique National}, + Year = {1997}, + + Url = {https://geodesie.ign.fr/contenu/fichiers/documentation/algorithmes/notice/NTG_88.pdf} +} + @TechReport{ONeilLaubscher1976, Title = {Extended Studies of a Quadrilateralized Spherical Cube Earth Data Base}, Author = {E. M. O'Neill and R. E. Laubscher}, diff --git a/docs/source/resource_files.rst b/docs/source/resource_files.rst index 13b9386d..ea02fd4b 100644 --- a/docs/source/resource_files.rst +++ b/docs/source/resource_files.rst @@ -40,6 +40,11 @@ The following paths are checked in order: PROJ installation is not moved somewhere else. - The current directory +When networking capabilities are enabled, either by API with the +:c:func:`proj_context_set_enable_network` function or when the +:envvar:`PROJ_NETWORK` environment variable is set to ``ON``, PROJ will +attempt to use remote grids stored on CDN (Content Delivery Network) storage. + .. _proj-db: proj.db @@ -49,6 +54,30 @@ A proj installation includes a SQLite database of transformation information that must be accessible for the library to work properly. The library will print an error if the database can't be found. +proj.ini +------------------------------------------------------------------------------- + +.. versionadded:: 7.0 + +proj.ini is a text configuration file, mostly dedicated at setting up network +related parameters. + +Its default content is: + +:: + + [general] + ; Lines starting by ; are commented lines. + ; + + ; Network capabilities disabled by default. + ; Can be overriden with the PROJ_NETWORK=ON environment variable. + ; network = on + + ; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable. + cdn_endpoint = https://cdn.proj.org + + Transformation grids ------------------------------------------------------------------------------- diff --git a/docs/source/usage/environmentvars.rst b/docs/source/usage/environmentvars.rst index 457432a0..24ae4a45 100644 --- a/docs/source/usage/environmentvars.rst +++ b/docs/source/usage/environmentvars.rst @@ -56,3 +56,21 @@ done by setting the variable with no content:: Set the debug level of PROJ. The default debug level is zero, which results in no debug output when using PROJ. A number from 1-3, whit 3 being the most verbose setting. + +.. envvar:: PROJ_NETWORK + + .. versionadded:: 7.0.0 + + If set to ON, enable the capability to use remote grids stored on CDN + (Content Delivery Network) storage, when grids are not available locally. + Alternatively, the :c:func:`proj_context_set_enable_network` function can + be used. + +.. envvar:: PROJ_NETWORK_ENDPOINT + + .. versionadded:: 7.0.0 + + Define the endpoint of the CDN storage. Normally defined through the proj.ini + configuration file locale in PROJ_LIB. + Alternatively, the :c:func:`proj_context_set_url_endpoint` function can + be used. diff --git a/include/proj/coordinateoperation.hpp b/include/proj/coordinateoperation.hpp index 55bc28a0..bee3eff2 100644 --- a/include/proj/coordinateoperation.hpp +++ b/include/proj/coordinateoperation.hpp @@ -144,10 +144,12 @@ class PROJ_GCC_DLL CoordinateOperation : public common::ObjectUsage, /** \brief Return grids needed by an operation. */ PROJ_DLL virtual std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const = 0; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const = 0; PROJ_DLL bool - isPROJInstantiable(const io::DatabaseContextPtr &databaseContext) const; + isPROJInstantiable(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const; PROJ_DLL bool hasBallparkTransformation() const; @@ -595,7 +597,8 @@ class PROJ_GCC_DLL SingleOperation : virtual public CoordinateOperation { std::vector<metadata::PositionalAccuracyNNPtr>()); PROJ_DLL std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; PROJ_DLL std::list<std::string> validateParameters() const; @@ -1672,7 +1675,8 @@ class PROJ_GCC_DLL ConcatenatedOperation final : public CoordinateOperation { bool checkExtent); // throw InvalidOperation PROJ_DLL std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; PROJ_PRIVATE : @@ -1801,6 +1805,12 @@ class PROJ_GCC_DLL CoordinateOperationContext { /** Ignore grid availability at all. Results will be presented as if * all grids were available. */ IGNORE_GRID_AVAILABILITY, + + /** Results will be presented as if grids known to PROJ (that is + * registered in the grid_alternatives table of its database) were + * available. Used typically when networking is enabled. + */ + KNOWN_AVAILABLE, }; PROJ_DLL void setGridAvailabilityUse(GridAvailabilityUse use); diff --git a/include/proj/internal/coordinateoperation_internal.hpp b/include/proj/internal/coordinateoperation_internal.hpp index 7ae2cd78..921021ec 100644 --- a/include/proj/internal/coordinateoperation_internal.hpp +++ b/include/proj/internal/coordinateoperation_internal.hpp @@ -177,8 +177,10 @@ class InverseConversion : public Conversion, public InverseCoordinateOperation { // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded' // via dominance std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override { - return SingleOperation::gridsNeeded(databaseContext); + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override { + return SingleOperation::gridsNeeded(databaseContext, + considerKnownGridsAsAvailable); } #endif @@ -227,8 +229,10 @@ class InverseTransformation : public Transformation, // 'osgeo::proj::operation::SingleOperation::osgeo::proj::operation::SingleOperation::gridsNeeded' // via dominance std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override { - return SingleOperation::gridsNeeded(databaseContext); + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override { + return SingleOperation::gridsNeeded(databaseContext, + considerKnownGridsAsAvailable); } #endif @@ -269,7 +273,8 @@ class PROJBasedOperation : public SingleOperation { bool hasRoughTransformation); std::set<GridDescription> - gridsNeeded(const io::DatabaseContextPtr &databaseContext) const override; + gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const override; protected: PROJBasedOperation(const PROJBasedOperation &) = default; diff --git a/include/proj/io.hpp b/include/proj/io.hpp index be293eb9..2839896e 100644 --- a/include/proj/io.hpp +++ b/include/proj/io.hpp @@ -832,6 +832,7 @@ class PROJ_GCC_DLL DatabaseContext { bool &inverse) const; PROJ_DLL bool lookForGridInfo(const std::string &projFilename, + bool considerKnownGridsAsAvailable, std::string &fullFilename, std::string &packageName, std::string &url, bool &directDownload, bool &openLicense, @@ -1050,7 +1051,8 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, bool tryReverseOrder = false, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + bool tryReverseOrder = false, bool reportOnlyIntersectingTransformations = false, const metadata::ExtentPtr &intersectingExtent1 = nullptr, const metadata::ExtentPtr &intersectingExtent2 = nullptr) const; @@ -1060,7 +1062,7 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector<std::pair<std::string, std::string>> &intermediateCRSAuthCodes, ObjectType allowedIntermediateObjectType = ObjectType::CRS, @@ -1126,7 +1128,7 @@ class PROJ_GCC_DLL AuthorityFactory { const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector<std::string> &allowedAuthorities, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const; diff --git a/scripts/doxygen.sh b/scripts/doxygen.sh index 48646237..9646eed8 100755 --- a/scripts/doxygen.sh +++ b/scripts/doxygen.sh @@ -34,7 +34,7 @@ fi mkdir -p docs/build/tmp_breathe python scripts/generate_breathe_friendly_general_doc.py rm -rf docs/build/xml/ -(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj src/proj.h docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen - > docs/build/tmp_breathe/docs_log.txt 2>&1 +(cat Doxyfile; printf "GENERATE_HTML=NO\nGENERATE_XML=YES\nINPUT= src/iso19111 include/proj src/proj.h src/filemanager.cpp docs/build/tmp_breathe/general_doc.dox.reworked.h") | doxygen - > docs/build/tmp_breathe/docs_log.txt 2>&1 if grep -i warning docs/build/tmp_breathe/docs_log.txt; then echo "Doxygen warnings found" && cat docs/build/tmp_breathe/docs_log.txt && /bin/false; else diff --git a/scripts/reference_exported_symbols.txt b/scripts/reference_exported_symbols.txt index e20f29b3..3e17a3c7 100644 --- a/scripts/reference_exported_symbols.txt +++ b/scripts/reference_exported_symbols.txt @@ -299,8 +299,8 @@ osgeo::proj::io::AuthorityFactory::create(dropbox::oxygen::nn<std::shared_ptr<os osgeo::proj::io::AuthorityFactory::createEllipsoid(std::string const&) const osgeo::proj::io::AuthorityFactory::createExtent(std::string const&) const osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&) const -osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, bool, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const -osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, std::vector<std::pair<std::string, std::string>, std::allocator<std::pair<std::string, std::string> > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector<std::string, std::allocator<std::string> > const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const +osgeo::proj::io::AuthorityFactory::createFromCoordinateReferenceSystemCodes(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, bool, bool, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const +osgeo::proj::io::AuthorityFactory::createFromCRSCodesWithIntermediates(std::string const&, std::string const&, std::string const&, std::string const&, bool, bool, bool, bool, std::vector<std::pair<std::string, std::string>, std::allocator<std::pair<std::string, std::string> > > const&, osgeo::proj::io::AuthorityFactory::ObjectType, std::vector<std::string, std::allocator<std::string> > const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&, std::shared_ptr<osgeo::proj::metadata::Extent> const&) const osgeo::proj::io::AuthorityFactory::createGeodeticCRS(std::string const&) const osgeo::proj::io::AuthorityFactory::createGeodeticDatum(std::string const&) const osgeo::proj::io::AuthorityFactory::createGeographicCRS(std::string const&) const @@ -330,7 +330,7 @@ osgeo::proj::io::DatabaseContext::getDatabaseStructure() const osgeo::proj::io::DatabaseContext::getMetadata(char const*) const osgeo::proj::io::DatabaseContext::getPath() const osgeo::proj::io::DatabaseContext::getSqliteHandle() const -osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, std::string&, std::string&, std::string&, bool&, bool&, bool&) const +osgeo::proj::io::DatabaseContext::lookForGridInfo(std::string const&, bool, std::string&, std::string&, std::string&, bool&, bool&, bool&) const osgeo::proj::io::FactoryException::~FactoryException() osgeo::proj::io::FactoryException::FactoryException(char const*) osgeo::proj::io::FactoryException::FactoryException(osgeo::proj::io::FactoryException const&) @@ -465,7 +465,7 @@ osgeo::proj::metadata::VerticalExtent::~VerticalExtent() osgeo::proj::operation::ConcatenatedOperation::~ConcatenatedOperation() osgeo::proj::operation::ConcatenatedOperation::createComputeMetadata(std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> > > > const&, bool) osgeo::proj::operation::ConcatenatedOperation::create(osgeo::proj::util::PropertyMap const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::operation::CoordinateOperation> > > > const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> > > > const&) -osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const +osgeo::proj::operation::ConcatenatedOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const osgeo::proj::operation::ConcatenatedOperation::inverse() const osgeo::proj::operation::ConcatenatedOperation::operations() const osgeo::proj::operation::Conversion::~Conversion() @@ -575,7 +575,7 @@ osgeo::proj::operation::CoordinateOperationFactory::createOperation(dropbox::oxy osgeo::proj::operation::CoordinateOperationFactory::createOperations(dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::crs::CRS> > const&, dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::crs::CRS> > const&, dropbox::oxygen::nn<std::unique_ptr<osgeo::proj::operation::CoordinateOperationContext, std::default_delete<osgeo::proj::operation::CoordinateOperationContext> > > const&) const osgeo::proj::operation::CoordinateOperation::hasBallparkTransformation() const osgeo::proj::operation::CoordinateOperation::interpolationCRS() const -osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const +osgeo::proj::operation::CoordinateOperation::isPROJInstantiable(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const osgeo::proj::operation::CoordinateOperation::normalizeForVisualization() const osgeo::proj::operation::CoordinateOperation::operationVersion() const osgeo::proj::operation::CoordinateOperation::shallowClone() const @@ -621,7 +621,7 @@ osgeo::proj::operation::ParameterValue::value() const osgeo::proj::operation::ParameterValue::valueFile() const osgeo::proj::operation::PointMotionOperation::~PointMotionOperation() osgeo::proj::operation::SingleOperation::createPROJBased(osgeo::proj::util::PropertyMap const&, std::string const&, std::shared_ptr<osgeo::proj::crs::CRS> const&, std::shared_ptr<osgeo::proj::crs::CRS> const&, std::vector<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> >, std::allocator<dropbox::oxygen::nn<std::shared_ptr<osgeo::proj::metadata::PositionalAccuracy> > > > const&) -osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&) const +osgeo::proj::operation::SingleOperation::gridsNeeded(std::shared_ptr<osgeo::proj::io::DatabaseContext> const&, bool) const osgeo::proj::operation::SingleOperation::method() const osgeo::proj::operation::SingleOperation::parameterValue(int) const osgeo::proj::operation::SingleOperation::parameterValueMeasure(int) const @@ -707,6 +707,7 @@ pj_chomp(char*) pj_cleanup_lock pj_clear_initcache pj_compare_datums +pj_context_is_network_enabled(projCtx_t*) pj_ctx_alloc pj_ctx_fclose pj_ctx_fgets @@ -794,9 +795,12 @@ proj_context_get_use_proj4_init_rules proj_context_guess_wkt_dialect proj_context_set_autoclose_database proj_context_set_database_path +proj_context_set_enable_network proj_context_set_file_finder +proj_context_set_network_callbacks proj_context_set(PJconsts*, projCtx_t*) proj_context_set_search_paths +proj_context_set_url_endpoint proj_context_use_proj4_init_rules proj_convert_conversion_to_other_method proj_coord diff --git a/scripts/reformat_cpp.sh b/scripts/reformat_cpp.sh index 899e665f..ca20e0d8 100755 --- a/scripts/reformat_cpp.sh +++ b/scripts/reformat_cpp.sh @@ -17,7 +17,8 @@ TOPDIR="$SCRIPT_DIR/.." for i in "$TOPDIR"/include/proj/*.hpp "$TOPDIR"/include/proj/internal/*.hpp \ "$TOPDIR"/src/iso19111/*.cpp "$TOPDIR"/test/unit/*.cpp "$TOPDIR"/src/apps/projinfo.cpp \ - "$TOPDIR"/src/tracing.cpp "$TOPDIR"/src/grids.hpp "$TOPDIR"/src/grids.cpp; do + "$TOPDIR"/src/tracing.cpp "$TOPDIR"/src/grids.hpp "$TOPDIR"/src/grids.cpp \ + "$TOPDIR"/src/filemanager.hpp "$TOPDIR"/src/filemanager.cpp ; do if ! echo "$i" | grep -q "lru_cache.hpp"; then "$SCRIPT_DIR"/reformat.sh "$i"; fi diff --git a/src/4D_api.cpp b/src/4D_api.cpp index efb4a86a..cee8262e 100644 --- a/src/4D_api.cpp +++ b/src/4D_api.cpp @@ -262,7 +262,7 @@ similarly, but prefers the 2D resp. 3D interfaces if available. auto coordOperation = dynamic_cast< NS_PROJ::operation::CoordinateOperation*>(alt.pj->iso_obj.get()); if( coordOperation ) { - if( coordOperation->gridsNeeded(dbContext).empty() ) { + if( coordOperation->gridsNeeded(dbContext, true).empty() ) { if( P->iCurCoordOp != i ) { std::string msg("Using coordinate operation "); msg += alt.name; @@ -1117,7 +1117,10 @@ PJ *proj_create_crs_to_crs_from_pj (PJ_CONTEXT *ctx, const PJ *source_crs, cons proj_operation_factory_context_set_spatial_criterion( ctx, operation_ctx, PROJ_SPATIAL_CRITERION_PARTIAL_INTERSECTION); proj_operation_factory_context_set_grid_availability_use( - ctx, operation_ctx, PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); + ctx, operation_ctx, + pj_context_is_network_enabled(ctx) ? + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + PROJ_GRID_AVAILABILITY_DISCARD_OPERATION_IF_MISSING_GRID); auto op_list = proj_create_operations(ctx, source_crs, target_crs, operation_ctx); diff --git a/src/Makefile.am b/src/Makefile.am index a12de4e1..afe4bcb7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ TESTS = geodtest check_PROGRAMS = geodtest AM_CPPFLAGS = -DPROJ_LIB=\"$(pkgdatadir)\" \ - -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ + -DMUTEX_@MUTEX_SETTING@ @JNI_INCLUDE@ -I$(top_srcdir)/include @SQLITE3_CFLAGS@ @TIFF_CFLAGS@ @TIFF_ENABLED_FLAGS@ @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ AM_CXXFLAGS = @CXX_WFLAGS@ @FLTO_FLAG@ include_HEADERS = proj.h proj_experimental.h proj_constants.h proj_api.h geodesic.h \ @@ -43,7 +43,7 @@ geodtest_LDADD = libproj.la lib_LTLIBRARIES = libproj.la libproj_la_LDFLAGS = -no-undefined -version-info 17:0:2 -libproj_la_LIBADD = @SQLITE3_LIBS@ +libproj_la_LIBADD = @SQLITE3_LIBS@ @TIFF_LIBS@ @CURL_LIBS@ libproj_la_SOURCES = \ pj_list.h proj_internal.h \ @@ -183,6 +183,7 @@ libproj_la_SOURCES = \ transformations/horner.cpp \ transformations/molodensky.cpp \ transformations/vgridshift.cpp \ + transformations/xyzgridshift.cpp \ \ aasincos.cpp adjlon.cpp \ dmstor.cpp auth.cpp \ @@ -214,7 +215,9 @@ libproj_la_SOURCES = \ tracing.cpp \ \ grids.hpp \ - grids.cpp + grids.cpp \ + filemanager.hpp \ + filemanager.cpp # The sed hack is to please MSVC diff --git a/src/apply_gridshift.cpp b/src/apply_gridshift.cpp index a0ffa394..4ef86fc0 100644 --- a/src/apply_gridshift.cpp +++ b/src/apply_gridshift.cpp @@ -36,6 +36,8 @@ #include <stdio.h> #include <string.h> +#include <limits> + #include "proj.h" #include "proj_internal.h" #include "proj/internal/internal.hpp" @@ -117,7 +119,7 @@ ListOfHGrids proj_hgrid_init(PJ* P, const char *gridkey) { typedef struct { pj_int32 lam, phi; } ILP; -static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { +static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid, bool compensateNTConvention) { PJ_LP val, frct; ILP indx; int in; @@ -162,10 +164,10 @@ static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { float f10Lon = 0, f10Lat = 0; float f01Lon = 0, f01Lat = 0; float f11Lon = 0, f11Lat = 0; - if( !grid->valueAt(indx.lam, indx.phi, f00Lon, f00Lat) || - !grid->valueAt(indx.lam + 1, indx.phi, f10Lon, f10Lat) || - !grid->valueAt(indx.lam, indx.phi + 1, f01Lon, f01Lat) || - !grid->valueAt(indx.lam + 1, indx.phi + 1, f11Lon, f11Lat) ) + if( !grid->valueAt(indx.lam, indx.phi, compensateNTConvention, f00Lon, f00Lat) || + !grid->valueAt(indx.lam + 1, indx.phi, compensateNTConvention, f10Lon, f10Lat) || + !grid->valueAt(indx.lam, indx.phi + 1, compensateNTConvention, f01Lon, f01Lat) || + !grid->valueAt(indx.lam + 1, indx.phi + 1, compensateNTConvention, f11Lon, f11Lat) ) { return val; } @@ -191,7 +193,8 @@ static PJ_LP nad_intr(PJ_LP t, const HorizontalShiftGrid* grid) { #define TOL 1e-12 static -PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { +PJ_LP nad_cvt(projCtx ctx, PJ_LP in, int inverse, const HorizontalShiftGrid* grid, + const ListOfHGrids& grids) { PJ_LP t, tb,del, dif; int i = MAX_ITERATIONS; const double toltol = TOL*TOL; @@ -201,42 +204,56 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { /* normalize input to ll origin */ tb = in; - const auto& extent = grid->extentAndRes(); - tb.lam -= extent.westLon; - tb.phi -= extent.southLat; + const auto* extent = &(grid->extentAndRes()); + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; tb.lam = adjlon (tb.lam - M_PI) + M_PI; - t = nad_intr (tb, grid); + t = nad_intr (tb, grid, true); if (t.lam == HUGE_VAL) return t; if (!inverse) { - in.lam -= t.lam; + in.lam += t.lam; in.phi += t.phi; return in; } - t.lam = tb.lam + t.lam; + t.lam = tb.lam - t.lam; t.phi = tb.phi - t.phi; do { - del = nad_intr(t, grid); - - /* This case used to return failure, but I have - changed it to return the first order approximation - of the inverse shift. This avoids cases where the - grid shift *into* this grid came from another grid. - While we aren't returning optimally correct results - I feel a close result in this case is better than - no result. NFW - To demonstrate use -112.5839956 49.4914451 against - the NTv2 grid shift file from Canada. */ + del = nad_intr(t, grid, true); + + /* We can possibly go outside of the initial guessed grid, so try */ + /* to fetch a new grid into which iterate... */ if (del.lam == HUGE_VAL) - break; + { + PJ_LP lp; + lp.lam = t.lam + extent->westLon; + lp.phi = t.phi + extent->southLat; + auto newGrid = findGrid(grids, lp); + if( newGrid == nullptr || newGrid == grid || newGrid->isNullGrid() ) + break; + pj_log(ctx, PJ_LOG_DEBUG_MINOR, "Switching from grid %s to grid %s", + grid->name().c_str(), + newGrid->name().c_str()); + grid = newGrid; + extent = &(grid->extentAndRes()); + t.lam = lp.lam - extent->westLon; + t.phi = lp.phi - extent->southLat; + tb = in; + tb.lam -= extent->westLon; + tb.phi -= extent->southLat; + tb.lam = adjlon (tb.lam - M_PI) + M_PI; + dif.lam = std::numeric_limits<double>::max(); + dif.phi = std::numeric_limits<double>::max(); + continue; + } - dif.lam = t.lam - del.lam - tb.lam; - dif.phi = t.phi + del.phi - tb.phi; + dif.lam = t.lam + del.lam - tb.lam; + dif.phi = t.phi + del.phi - tb.phi; t.lam -= dif.lam; t.phi -= dif.phi; @@ -254,8 +271,8 @@ PJ_LP nad_cvt(PJ_LP in, int inverse, const HorizontalShiftGrid* grid) { if (del.lam==HUGE_VAL && getenv ("PROJ_DEBUG")) fprintf (stderr, "Inverse grid shift iteration failed, presumably at grid edge.\nUsing first approximation.\n"); - in.lam = adjlon (t.lam + extent.westLon); - in.phi = t.phi + extent.southLat; + in.lam = adjlon (t.lam + extent->westLon); + in.phi = t.phi + extent->southLat; return in; } @@ -280,7 +297,7 @@ PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids& grids, PJ_LP lp) { lp.lam = adjlon(lp.lam - M_PI) + M_PI; - out = nad_intr(lp, grid); + out = nad_intr(lp, grid, false); if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) { pj_ctx_set_errno(P->ctx, PJD_ERR_GRID_AREA); @@ -311,7 +328,7 @@ PJ_LP proj_hgrid_apply_internal(PJ_CONTEXT* ctx, } int inverse = direction == PJ_FWD ? 0 : 1; - out = nad_cvt(lp, inverse, grid); + out = nad_cvt(ctx, lp, inverse, grid, grids); if (out.lam == HUGE_VAL || out.phi == HUGE_VAL) pj_ctx_set_errno(ctx, PJD_ERR_GRID_AREA); diff --git a/src/apps/projinfo.cpp b/src/apps/projinfo.cpp index fd9b2f46..f13e526b 100644 --- a/src/apps/projinfo.cpp +++ b/src/apps/projinfo.cpp @@ -87,7 +87,8 @@ static void usage() { << " [--spatial-test contains|intersects]" << std::endl << " [--crs-extent-use none|both|intersection|smallest]" << std::endl - << " [--grid-check none|discard_missing|sort] " + << " [--grid-check " + "none|discard_missing|sort|known_available] " "[--show-superseded]" << std::endl << " [--pivot-crs always|if_no_direct_transformation|" @@ -509,7 +510,7 @@ static void outputObject( auto op = dynamic_cast<CoordinateOperation *>(obj.get()); if (op && dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); bool firstWarning = true; for (const auto &grid : setGrids) { if (!grid.available) { @@ -525,6 +526,7 @@ static void outputObject( if (!grid.url.empty()) { std::cout << " at " << grid.url; } + std::cout << ", or on CDN"; } else if (!grid.url.empty()) { std::cout << " Can be obtained at " << grid.url; } @@ -539,8 +541,9 @@ static void outputObject( // --------------------------------------------------------------------------- -static void outputOperationSummary(const CoordinateOperationNNPtr &op, - const DatabaseContextPtr &dbContext) { +static void outputOperationSummary( + const CoordinateOperationNNPtr &op, const DatabaseContextPtr &dbContext, + CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse) { auto ids = op->identifiers(); if (!ids.empty()) { std::cout << *(ids[0]->codeSpace()) << ":" << ids[0]->code(); @@ -586,10 +589,16 @@ static void outputOperationSummary(const CoordinateOperationNNPtr &op, if (dbContext && getenv("PROJINFO_NO_GRID_CHECK") == nullptr) { try { - auto setGrids = op->gridsNeeded(dbContext); + auto setGrids = op->gridsNeeded(dbContext, false); for (const auto &grid : setGrids) { if (!grid.available) { std::cout << ", at least one grid missing"; + if (gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE && + !grid.packageName.empty()) { + std::cout << " on the system, but available on CDN"; + } break; } } @@ -686,7 +695,7 @@ static void outputOperations( } if (summary) { for (const auto &op : list) { - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); } } else { bool first = true; @@ -701,7 +710,7 @@ static void outputOperations( "\xC2\xB0" << (i + 1) << ":" << std::endl << std::endl; - outputOperationSummary(op, dbContext); + outputOperationSummary(op, dbContext, gridAvailabilityUse); std::cout << std::endl; outputObject(dbContext, op, allowUseIntermediateCRS, outputOpt); } @@ -734,7 +743,9 @@ int main(int argc, char **argv) { CoordinateOperationContext::SourceTargetCRSExtentUse::SMALLEST; bool buildBoundCRSToWGS84 = false; CoordinateOperationContext::GridAvailabilityUse gridAvailabilityUse = - CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; + pj_context_is_network_enabled(nullptr) + ? CoordinateOperationContext::GridAvailabilityUse::KNOWN_AVAILABLE + : CoordinateOperationContext::GridAvailabilityUse::USE_FOR_SORTING; CoordinateOperationContext::IntermediateCRSUse allowUseIntermediateCRS = CoordinateOperationContext::IntermediateCRSUse:: IF_NO_DIRECT_TRANSFORMATION; @@ -947,6 +958,9 @@ int main(int argc, char **argv) { } else if (ci_equal(value, "sort")) { gridAvailabilityUse = CoordinateOperationContext:: GridAvailabilityUse::USE_FOR_SORTING; + } else if (ci_equal(value, "known_available")) { + gridAvailabilityUse = CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE; } else { std::cerr << "Unrecognized value for option --grid-check: " << value << std::endl; diff --git a/src/ctx.cpp b/src/ctx.cpp index bcb6e1cc..6172b3c8 100644 --- a/src/ctx.cpp +++ b/src/ctx.cpp @@ -33,6 +33,7 @@ #include "proj_experimental.h" #include "proj_internal.h" +#include "filemanager.hpp" /************************************************************************/ /* pj_get_ctx() */ @@ -60,9 +61,9 @@ void pj_set_ctx( projPJ pj, projCtx ctx ) if (pj==nullptr) return; pj->ctx = ctx; - if( pj->is_pipeline ) + if( pj->reassign_context ) { - pj_pipeline_assign_context_to_steps(pj, ctx); + pj->reassign_context(pj, ctx); } for( const auto &alt: pj->alternativeCoordinateOperations ) { @@ -95,7 +96,8 @@ projCtx_t projCtx_t::createDefault() projCtx_t ctx; ctx.debug_level = PJ_LOG_NONE; ctx.logger = pj_stderr_logger; - ctx.fileapi = pj_get_default_fileapi(); + ctx.fileapi_legacy = pj_get_default_fileapi(); + NS_PROJ::FileManager::fillDefaultNetworkInterface(&ctx); if( getenv("PROJ_DEBUG") != nullptr ) { @@ -133,12 +135,13 @@ projCtx_t::projCtx_t(const projCtx_t& other) debug_level = other.debug_level; logger = other.logger; logger_app_data = other.logger_app_data; - fileapi = other.fileapi; + fileapi_legacy = other.fileapi_legacy; epsg_file_exists = other.epsg_file_exists; set_search_paths(other.search_paths); file_finder = other.file_finder; file_finder_legacy = other.file_finder_legacy; file_finder_user_data = other.file_finder_user_data; + networking = other.networking; } /************************************************************************/ @@ -258,28 +261,3 @@ void *pj_ctx_get_app_data( projCtx ctx ) return nullptr; return ctx->logger_app_data; } - -/************************************************************************/ -/* pj_ctx_set_fileapi() */ -/************************************************************************/ - -void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) - -{ - if (nullptr==ctx) - return; - ctx->fileapi = fileapi; -} - -/************************************************************************/ -/* pj_ctx_get_fileapi() */ -/************************************************************************/ - -projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) - -{ - if (nullptr==ctx) - return nullptr; - return ctx->fileapi; -} - diff --git a/src/fileapi.cpp b/src/fileapi.cpp index 70c7b5de..70be2502 100644 --- a/src/fileapi.cpp +++ b/src/fileapi.cpp @@ -34,6 +34,7 @@ #include "proj.h" #include "proj_internal.h" +#include "filemanager.hpp" static PAFile stdio_fopen(projCtx ctx, const char *filename, const char *access); @@ -141,7 +142,7 @@ static void stdio_fclose(PAFile file) PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) { - return ctx->fileapi->FOpen(ctx, filename, access); + return ctx->fileapi_legacy->FOpen(ctx, filename, access); } /************************************************************************/ @@ -149,7 +150,7 @@ PAFile pj_ctx_fopen(projCtx ctx, const char *filename, const char *access) /************************************************************************/ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile file) { - return ctx->fileapi->FRead(buffer, size, nmemb, file); + return ctx->fileapi_legacy->FRead(buffer, size, nmemb, file); } /************************************************************************/ @@ -157,7 +158,7 @@ size_t pj_ctx_fread(projCtx ctx, void *buffer, size_t size, size_t nmemb, PAFile /************************************************************************/ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) { - return ctx->fileapi->FSeek(file, offset, whence); + return ctx->fileapi_legacy->FSeek(file, offset, whence); } /************************************************************************/ @@ -165,7 +166,7 @@ int pj_ctx_fseek(projCtx ctx, PAFile file, long offset, int whence) /************************************************************************/ long pj_ctx_ftell(projCtx ctx, PAFile file) { - return ctx->fileapi->FTell(file); + return ctx->fileapi_legacy->FTell(file); } /************************************************************************/ @@ -173,7 +174,7 @@ long pj_ctx_ftell(projCtx ctx, PAFile file) /************************************************************************/ void pj_ctx_fclose(projCtx ctx, PAFile file) { - ctx->fileapi->FClose(file); + ctx->fileapi_legacy->FClose(file); } /************************************************************************/ @@ -212,3 +213,28 @@ char *pj_ctx_fgets(projCtx ctx, char *line, int size, PAFile file) } return line; } + +/************************************************************************/ +/* pj_ctx_set_fileapi() */ +/************************************************************************/ + +void pj_ctx_set_fileapi( projCtx ctx, projFileAPI *fileapi ) + +{ + if (nullptr==ctx) + return; + ctx->fileapi_legacy = fileapi; +} + +/************************************************************************/ +/* pj_ctx_get_fileapi() */ +/************************************************************************/ + +projFileAPI *pj_ctx_get_fileapi( projCtx ctx ) + +{ + if (nullptr==ctx) + return nullptr; + return ctx->fileapi_legacy; +} + diff --git a/src/filemanager.cpp b/src/filemanager.cpp new file mode 100644 index 00000000..625dca03 --- /dev/null +++ b/src/filemanager.cpp @@ -0,0 +1,997 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FROM_PROJ_CPP +#define FROM_PROJ_CPP +#endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS + +#include <algorithm> +#include <functional> +#include <limits> + +#include "filemanager.hpp" +#include "proj.h" +#include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" +#include "proj_internal.h" + +#ifdef __MINGW32__ +// mingw32-win32 doesn't implement std::mutex +namespace { +class MyMutex { + public: + // cppcheck-suppress functionStatic + void lock() { pj_acquire_lock(); } + // cppcheck-suppress functionStatic + void unlock() { pj_release_lock(); } +}; +} +#else +#include <mutex> +#define MyMutex std::mutex +#endif + +#ifdef CURL_ENABLED +#include <curl/curl.h> +#include <sqlite3.h> // for sqlite3_snprintf +#endif + +#if defined(__linux) +#include <unistd.h> +#elif defined(_WIN32) +#include <windows.h> +#elif defined(__MACH__) && defined(__APPLE__) +#include <mach-o/dyld.h> +#elif defined(__FreeBSD__) +#include <sys/sysctl.h> +#include <sys/types.h> +#endif + +//! @cond Doxygen_Suppress + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +using namespace NS_PROJ::internal; + +NS_PROJ_START + +// --------------------------------------------------------------------------- + +File::File(const std::string &name) : name_(name) {} + +// --------------------------------------------------------------------------- + +File::~File() = default; + +// --------------------------------------------------------------------------- + +class FileStdio : public File { + PJ_CONTEXT *m_ctx; + FILE *m_fp; + + FileStdio(const FileStdio &) = delete; + FileStdio &operator=(const FileStdio &) = delete; + + protected: + FileStdio(const std::string &name, PJ_CONTEXT *ctx, FILE *fp) + : File(name), m_ctx(ctx), m_fp(fp) {} + + public: + ~FileStdio() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileStdio::~FileStdio() { fclose(m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileStdio::read(void *buffer, size_t sizeBytes) { + return fread(buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileStdio::seek(unsigned long long offset, int whence) { + // TODO one day: use 64-bit offset compatible API + if (offset != static_cast<unsigned long long>(static_cast<long>(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return fseek(m_fp, static_cast<long>(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileStdio::tell() { + // TODO one day: use 64-bit offset compatible API + return ftell(m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> FileStdio::open(PJ_CONTEXT *ctx, const char *filename) { + auto fp = fopen(filename, "rb"); + return std::unique_ptr<File>(fp ? new FileStdio(filename, ctx, fp) + : nullptr); +} + +// --------------------------------------------------------------------------- + +#ifndef REMOVE_LEGACY_SUPPORT + +class FileLegacyAdapter : public File { + PJ_CONTEXT *m_ctx; + PAFile m_fp; + + FileLegacyAdapter(const FileLegacyAdapter &) = delete; + FileLegacyAdapter &operator=(const FileLegacyAdapter &) = delete; + + protected: + FileLegacyAdapter(const std::string &name, PJ_CONTEXT *ctx, PAFile fp) + : File(name), m_ctx(ctx), m_fp(fp) {} + + public: + ~FileLegacyAdapter() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +FileLegacyAdapter::~FileLegacyAdapter() { pj_ctx_fclose(m_ctx, m_fp); } + +// --------------------------------------------------------------------------- + +size_t FileLegacyAdapter::read(void *buffer, size_t sizeBytes) { + return pj_ctx_fread(m_ctx, buffer, 1, sizeBytes, m_fp); +} + +// --------------------------------------------------------------------------- + +bool FileLegacyAdapter::seek(unsigned long long offset, int whence) { + if (offset != static_cast<unsigned long long>(static_cast<long>(offset))) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Attempt at seeking to a 64 bit offset. Not supported yet"); + return false; + } + return pj_ctx_fseek(m_ctx, m_fp, static_cast<long>(offset), whence) == 0; +} + +// --------------------------------------------------------------------------- + +unsigned long long FileLegacyAdapter::tell() { + return pj_ctx_ftell(m_ctx, m_fp); +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> FileLegacyAdapter::open(PJ_CONTEXT *ctx, + const char *filename) { + auto fid = pj_ctx_fopen(ctx, filename, "rb"); + return std::unique_ptr<File>(fid ? new FileLegacyAdapter(filename, ctx, fid) + : nullptr); +} + +#endif // REMOVE_LEGACY_SUPPORT + +// --------------------------------------------------------------------------- + +constexpr size_t DOWNLOAD_CHUNK_SIZE = 16 * 1024; +constexpr int MAX_CHUNKS = 64; + +class NetworkChunkCache { + public: + void insert(const std::string &url, unsigned long long chunkIdx, + std::vector<unsigned char> &&data); + + std::shared_ptr<std::vector<unsigned char>> + get(const std::string &url, unsigned long long chunkIdx); + + void clear(); + + private: + struct Key { + std::string url; + unsigned long long chunkIdx; + + Key(const std::string &urlIn, unsigned long long chunkIdxIn) + : url(urlIn), chunkIdx(chunkIdxIn) {} + bool operator==(const Key &other) const { + return url == other.url && chunkIdx == other.chunkIdx; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return std::hash<std::string>{}(k.url) ^ + (std::hash<unsigned long long>{}(k.chunkIdx) << 1); + } + }; + + lru11::Cache< + Key, std::shared_ptr<std::vector<unsigned char>>, MyMutex, + std::unordered_map< + Key, + typename std::list<lru11::KeyValuePair< + Key, std::shared_ptr<std::vector<unsigned char>>>>::iterator, + KeyHasher>> + cache_{MAX_CHUNKS}; +}; + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::insert(const std::string &url, + unsigned long long chunkIdx, + std::vector<unsigned char> &&data) { + cache_.insert( + Key(url, chunkIdx), + std::make_shared<std::vector<unsigned char>>(std::move(data))); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr<std::vector<unsigned char>> +NetworkChunkCache::get(const std::string &url, unsigned long long chunkIdx) { + std::shared_ptr<std::vector<unsigned char>> ret; + cache_.tryGet(Key(url, chunkIdx), ret); + return ret; +} + +// --------------------------------------------------------------------------- + +void NetworkChunkCache::clear() { cache_.clear(); } + +// --------------------------------------------------------------------------- + +static NetworkChunkCache gNetworkChunkCache{}; + +struct FileProperties { + unsigned long long size; +}; + +static lru11::Cache<std::string, FileProperties, MyMutex> + gNetworkFileProperties{}; + +// --------------------------------------------------------------------------- + +class NetworkFile : public File { + PJ_CONTEXT *m_ctx; + std::string m_url; + PROJ_NETWORK_HANDLE *m_handle; + unsigned long long m_pos = 0; + size_t m_nBlocksToDownload = 1; + unsigned long long m_lastDownloadedOffset; + unsigned long long m_filesize; + proj_network_close_cbk_type m_closeCbk; + + NetworkFile(const NetworkFile &) = delete; + NetworkFile &operator=(const NetworkFile &) = delete; + + protected: + NetworkFile(PJ_CONTEXT *ctx, const std::string &url, + PROJ_NETWORK_HANDLE *handle, + unsigned long long lastDownloadOffset, + unsigned long long filesize) + : File(url), m_ctx(ctx), m_url(url), m_handle(handle), + m_lastDownloadedOffset(lastDownloadOffset), m_filesize(filesize), + m_closeCbk(ctx->networking.close) {} + + public: + ~NetworkFile() override; + + size_t read(void *buffer, size_t sizeBytes) override; + bool seek(unsigned long long offset, int whence) override; + unsigned long long tell() override; + void reassign_context(PJ_CONTEXT *ctx) override; + + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename); +}; + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> NetworkFile::open(PJ_CONTEXT *ctx, const char *filename) { + if (gNetworkChunkCache.get(filename, 0)) { + unsigned long long filesize = 0; + FileProperties props; + if (gNetworkFileProperties.tryGet(filename, props)) { + filesize = props.size; + } + return std::unique_ptr<File>(new NetworkFile( + ctx, filename, nullptr, + std::numeric_limits<unsigned long long>::max(), filesize)); + } else { + std::vector<unsigned char> buffer(DOWNLOAD_CHUNK_SIZE); + size_t size_read = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + + auto handle = ctx->networking.open( + ctx, filename, 0, buffer.size(), &buffer[0], &size_read, + errorBuffer.size(), &errorBuffer[0], ctx->networking.user_data); + buffer.resize(size_read); + gNetworkChunkCache.insert(filename, 0, std::move(buffer)); + if (!handle) { + errorBuffer.resize(strlen(errorBuffer.data())); + pj_log(ctx, PJ_LOG_ERROR, "Cannot open %s: %s", filename, + errorBuffer.c_str()); + } + + unsigned long long filesize = 0; + if (handle) { + const char *contentRange = ctx->networking.get_header_value( + ctx, handle, "Content-Range", ctx->networking.user_data); + if (contentRange) { + const char *slash = strchr(contentRange, '/'); + if (slash) { + filesize = std::stoull(slash + 1); + FileProperties props; + props.size = filesize; + gNetworkFileProperties.insert(filename, props); + } + } + } + + return std::unique_ptr<File>( + handle ? new NetworkFile(ctx, filename, handle, size_read, filesize) + : nullptr); + } +} + +// --------------------------------------------------------------------------- + +size_t NetworkFile::read(void *buffer, size_t sizeBytes) { + + if (sizeBytes == 0) + return 0; + + auto iterOffset = m_pos; + while (sizeBytes) { + const auto chunkIdxToDownload = iterOffset / DOWNLOAD_CHUNK_SIZE; + const auto offsetToDownload = chunkIdxToDownload * DOWNLOAD_CHUNK_SIZE; + std::vector<unsigned char> region; + auto pChunk = gNetworkChunkCache.get(m_url, chunkIdxToDownload); + if (pChunk != nullptr) { + region = *pChunk; + } else { + if (offsetToDownload == m_lastDownloadedOffset) { + // In case of consecutive reads (of small size), we use a + // heuristic that we will read the file sequentially, so + // we double the requested size to decrease the number of + // client/server roundtrips. + if (m_nBlocksToDownload < 100) + m_nBlocksToDownload *= 2; + } else { + // Random reads. Cancel the above heuristics. + m_nBlocksToDownload = 1; + } + + // Ensure that we will request at least the number of blocks + // to satisfy the remaining buffer size to read. + const auto endOffsetToDownload = + ((iterOffset + sizeBytes + DOWNLOAD_CHUNK_SIZE - 1) / + DOWNLOAD_CHUNK_SIZE) * + DOWNLOAD_CHUNK_SIZE; + const auto nMinBlocksToDownload = static_cast<size_t>( + (endOffsetToDownload - offsetToDownload) / DOWNLOAD_CHUNK_SIZE); + if (m_nBlocksToDownload < nMinBlocksToDownload) + m_nBlocksToDownload = nMinBlocksToDownload; + + // Avoid reading already cached data. + // Note: this might get evicted if concurrent reads are done, but + // this should not cause bugs. Just missed optimization. + for (size_t i = 1; i < m_nBlocksToDownload; i++) { + if (gNetworkChunkCache.get(m_url, chunkIdxToDownload + i) != + nullptr) { + m_nBlocksToDownload = i; + break; + } + } + + if (m_nBlocksToDownload > MAX_CHUNKS) + m_nBlocksToDownload = MAX_CHUNKS; + + region.resize(m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE); + size_t nRead = 0; + std::string errorBuffer; + errorBuffer.resize(1024); + if (!m_handle) { + m_handle = m_ctx->networking.open( + m_ctx, m_url.c_str(), offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + &nRead, errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); + if (!m_handle) { + return 0; + } + } else { + nRead = m_ctx->networking.read_range( + m_ctx, m_handle, offsetToDownload, + m_nBlocksToDownload * DOWNLOAD_CHUNK_SIZE, ®ion[0], + errorBuffer.size(), &errorBuffer[0], + m_ctx->networking.user_data); + } + if (nRead == 0) { + errorBuffer.resize(strlen(errorBuffer.data())); + if (!errorBuffer.empty()) { + pj_log(m_ctx, PJ_LOG_ERROR, "Cannot read in %s: %s", + m_url.c_str(), errorBuffer.c_str()); + } + return 0; + } + region.resize(nRead); + m_lastDownloadedOffset = offsetToDownload + nRead; + + const auto nChunks = + (region.size() + DOWNLOAD_CHUNK_SIZE - 1) / DOWNLOAD_CHUNK_SIZE; + for (size_t i = 0; i < nChunks; i++) { + std::vector<unsigned char> chunk( + region.data() + i * DOWNLOAD_CHUNK_SIZE, + region.data() + + std::min((i + 1) * DOWNLOAD_CHUNK_SIZE, region.size())); + gNetworkChunkCache.insert(m_url, chunkIdxToDownload + i, + std::move(chunk)); + } + } + const size_t nToCopy = static_cast<size_t>( + std::min(static_cast<unsigned long long>(sizeBytes), + region.size() - (iterOffset - offsetToDownload))); + memcpy(buffer, region.data() + iterOffset - offsetToDownload, nToCopy); + buffer = static_cast<char *>(buffer) + nToCopy; + iterOffset += nToCopy; + sizeBytes -= nToCopy; + if (region.size() < static_cast<size_t>(DOWNLOAD_CHUNK_SIZE) && + sizeBytes != 0) { + break; + } + } + + size_t nRead = static_cast<size_t>(iterOffset - m_pos); + m_pos = iterOffset; + return nRead; +} + +// --------------------------------------------------------------------------- + +bool NetworkFile::seek(unsigned long long offset, int whence) { + if (whence == SEEK_SET) { + m_pos = offset; + } else if (whence == SEEK_CUR) { + m_pos += offset; + } else { + if (offset != 0) + return false; + m_pos = m_filesize; + } + return true; +} + +// --------------------------------------------------------------------------- + +unsigned long long NetworkFile::tell() { return m_pos; } + +// --------------------------------------------------------------------------- + +NetworkFile::~NetworkFile() { + if (m_handle) { + m_ctx->networking.close(m_ctx, m_handle, m_ctx->networking.user_data); + } +} + +// --------------------------------------------------------------------------- + +void NetworkFile::reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + if (m_closeCbk != m_ctx->networking.close) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Networking close callback has changed following context " + "reassignment ! This is highly suspicious"); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename) { +#ifndef REMOVE_LEGACY_SUPPORT + // If the user has specified a legacy fileapi, use it + if (ctx->fileapi_legacy != pj_get_default_fileapi()) { + return FileLegacyAdapter::open(ctx, filename); + } +#endif + if (starts_with(filename, "http://") || starts_with(filename, "https://")) { + if (!pj_context_is_network_enabled(ctx)) { + pj_log( + ctx, PJ_LOG_ERROR, + "Attempt at accessing remote resource not authorized. Either " + "set PROJ_NETWORK=ON or " + "proj_context_set_enable_network(ctx, TRUE)"); + return nullptr; + } + return NetworkFile::open(ctx, filename); + } + return FileStdio::open(ctx, filename); +} + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +struct CurlFileHandle { + std::string m_url; + CURL *m_handle; + std::string m_headers{}; + std::string m_lastval{}; + std::string m_useragent{}; + char m_szCurlErrBuf[CURL_ERROR_SIZE + 1] = {}; + + CurlFileHandle(const CurlFileHandle &) = delete; + CurlFileHandle &operator=(const CurlFileHandle &) = delete; + + explicit CurlFileHandle(const char *url, CURL *handle); + ~CurlFileHandle(); + + static PROJ_NETWORK_HANDLE * + open(PJ_CONTEXT *, const char *url, unsigned long long offset, + size_t size_to_read, void *buffer, size_t *out_size_read, + size_t error_string_max_size, char *out_error_string, void *); +}; + +// --------------------------------------------------------------------------- + +static std::string GetExecutableName() { +#if defined(__linux) + std::string path; + path.resize(1024); + const auto ret = readlink("/proc/self/exe", &path[0], path.size()); + if (ret > 0) { + path.resize(ret); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(_WIN32) + std::string path; + path.resize(1024); + if (GetModuleFileNameA(nullptr, &path[0], + static_cast<DWORD>(path.size()))) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('\\'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__MACH__) && defined(__APPLE__) + std::string path; + path.resize(1024); + uint32_t size = static_cast<uint32_t>(path.size()); + if (_NSGetExecutablePath(&path[0], &size) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#elif defined(__FreeBSD__) + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PATHNAME; + mib[3] = -1; + std::string path; + path.resize(1024); + size_t size = path.size(); + if (sysctl(mib, 4, &path[0], &size, nullptr, 0) == 0) { + path.resize(strlen(path.c_str())); + const auto pos = path.rfind('/'); + if (pos != std::string::npos) { + path = path.substr(pos + 1); + } + return path; + } +#endif + + return std::string(); +} + +// --------------------------------------------------------------------------- + +CurlFileHandle::CurlFileHandle(const char *url, CURL *handle) + : m_url(url), m_handle(handle) { + curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); + + if (getenv("PROJ_CURL_VERBOSE")) + curl_easy_setopt(handle, CURLOPT_VERBOSE, 1); + +// CURLOPT_SUPPRESS_CONNECT_HEADERS is defined in curl 7.54.0 or newer. +#if LIBCURL_VERSION_NUM >= 0x073600 + curl_easy_setopt(handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L); +#endif + + // Enable following redirections. Requires libcurl 7.10.1 at least. + curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10); + + if (getenv("PROJ_UNSAFE_SSL")) { + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 0L); + } + + curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, m_szCurlErrBuf); + + if (getenv("PROJ_NO_USERAGENT") == nullptr) { + m_useragent = "PROJ " STR(PROJ_VERSION_MAJOR) "." STR( + PROJ_VERSION_MINOR) "." STR(PROJ_VERSION_PATCH); + const auto exeName = GetExecutableName(); + if (!exeName.empty()) { + m_useragent = exeName + " using " + m_useragent; + } + curl_easy_setopt(handle, CURLOPT_USERAGENT, m_useragent.data()); + } +} + +// --------------------------------------------------------------------------- + +CurlFileHandle::~CurlFileHandle() { curl_easy_cleanup(m_handle); } + +// --------------------------------------------------------------------------- + +static size_t pj_curl_write_func(void *buffer, size_t count, size_t nmemb, + void *req) { + const size_t nSize = count * nmemb; + auto pStr = static_cast<std::string *>(req); + if (pStr->size() + nSize > pStr->capacity()) { + // to avoid servers not honouring Range to cause excessive memory + // allocation + return 0; + } + pStr->append(static_cast<const char *>(buffer), nSize); + return nmemb; +} + +// --------------------------------------------------------------------------- + +PROJ_NETWORK_HANDLE *CurlFileHandle::open(PJ_CONTEXT *, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return nullptr; + + auto file = + std::unique_ptr<CurlFileHandle>(new CurlFileHandle(url, hCurlHandle)); + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string headers; + headers.reserve(16 * 1024); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &headers); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, pj_curl_write_func); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + file->m_szCurlErrBuf[0] = '\0'; + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, nullptr); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (file->m_szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + file->m_szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } + return nullptr; + } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + *out_size_read = std::min(size_to_read, body.size()); + + file->m_headers = std::move(headers); + return reinterpret_cast<PROJ_NETWORK_HANDLE *>(file.release()); +} + +// --------------------------------------------------------------------------- + +static void pj_curl_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *handle, + void * /*user_data*/) { + delete reinterpret_cast<CurlFileHandle *>(handle); +} + +// --------------------------------------------------------------------------- + +static size_t pj_curl_read_range(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *raw_handle, + unsigned long long offset, size_t size_to_read, + void *buffer, size_t error_string_max_size, + char *out_error_string, void *) { + auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle); + auto hCurlHandle = handle->m_handle; + + char szBuffer[128]; + sqlite3_snprintf(sizeof(szBuffer), szBuffer, "%llu-%llu", offset, + offset + size_to_read - 1); + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, szBuffer); + + std::string body; + body.reserve(size_to_read); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, pj_curl_write_func); + + handle->m_szCurlErrBuf[0] = '\0'; + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, nullptr); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, nullptr); + + if (response_code == 0 || response_code >= 300) { + if (out_error_string) { + if (handle->m_szCurlErrBuf[0]) { + snprintf(out_error_string, error_string_max_size, "%s", + handle->m_szCurlErrBuf); + } else { + snprintf(out_error_string, error_string_max_size, + "HTTP error %ld: %s", response_code, body.c_str()); + } + } + return 0; + } + if (out_error_string && error_string_max_size) { + out_error_string[0] = '\0'; + } + + if (!body.empty()) { + memcpy(buffer, body.data(), std::min(size_to_read, body.size())); + } + return std::min(size_to_read, body.size()); +} + +// --------------------------------------------------------------------------- + +static const char *pj_curl_get_header_value(PJ_CONTEXT *, + PROJ_NETWORK_HANDLE *raw_handle, + const char *header_name, void *) { + auto handle = reinterpret_cast<CurlFileHandle *>(raw_handle); + auto pos = ci_find(handle->m_headers, header_name); + if (pos == std::string::npos) + return nullptr; + const char *c_str = handle->m_headers.c_str(); + if (c_str[pos] == ':') + pos++; + while (c_str[pos] == ' ') + pos++; + auto posEnd = pos; + while (c_str[posEnd] != '\n' && c_str[posEnd] != '\0') + posEnd++; + handle->m_lastval = handle->m_headers.substr(pos, posEnd - pos); + return handle->m_lastval.c_str(); +} + +#else + +// --------------------------------------------------------------------------- + +static PROJ_NETWORK_HANDLE * +no_op_network_open(PJ_CONTEXT *ctx, const char * /* url */, + unsigned long long, /* offset */ + size_t, /* size to read */ + void *, /* buffer to update with bytes read*/ + size_t *, /* output: size actually read */ + size_t error_string_max_size, char *out_error_string, + void * /*user_data*/) { + if (out_error_string) { + snprintf(out_error_string, error_string_max_size, "%s", + "Network functionality not available"); + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static void no_op_network_close(PJ_CONTEXT *, PROJ_NETWORK_HANDLE *, + void * /*user_data*/) {} + +#endif + +// --------------------------------------------------------------------------- + +void FileManager::fillDefaultNetworkInterface(PJ_CONTEXT *ctx) { +#ifdef CURL_ENABLED + ctx->networking.open = CurlFileHandle::open; + ctx->networking.close = pj_curl_close; + ctx->networking.read_range = pj_curl_read_range; + ctx->networking.get_header_value = pj_curl_get_header_value; +#else + ctx->networking.open = no_op_network_open; + ctx->networking.close = no_op_network_close; +#endif +} + +// --------------------------------------------------------------------------- + +void FileManager::clearCache() { + gNetworkChunkCache.clear(); + gNetworkFileProperties.clear(); +} + +// --------------------------------------------------------------------------- + +NS_PROJ_END + +//! @endcond + +// --------------------------------------------------------------------------- + +/** Define a custom set of callbacks for network access. + * + * All callbacks should be provided (non NULL pointers). + * + * @param ctx PROJ context, or NULL + * @param open_cbk Callback to open a remote file given its URL + * @param close_cbk Callback to close a remote file. + * @param get_header_value_cbk Callback to get HTTP headers + * @param read_range_cbk Callback to read a range of bytes inside a remote file. + * @param user_data Arbitrary pointer provided by the user, and passed to the + * above callbacks. May be NULL. + * @return TRUE in case of success. + */ +int proj_context_set_network_callbacks( + PJ_CONTEXT *ctx, proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_read_range_type read_range_cbk, void *user_data) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (!open_cbk || !close_cbk || !get_header_value_cbk || !read_range_cbk) { + return false; + } + ctx->networking.open = open_cbk; + ctx->networking.close = close_cbk; + ctx->networking.get_header_value = get_header_value_cbk; + ctx->networking.read_range = read_range_cbk; + ctx->networking.user_data = user_data; + return true; +} + +// --------------------------------------------------------------------------- + +/** Enable or disable network access. +* +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK environment variable. +* +* @param ctx PROJ context, or NULL +* @param enable TRUE if network access is allowed. +* @return TRUE if network access is possible. That is either libcurl is +* available, or an alternate interface has been set. +*/ +int proj_context_set_enable_network(PJ_CONTEXT *ctx, int enable) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); + ctx->networking.enabled_env_variable_checked = true; + ctx->networking.enabled = enable != FALSE; +#ifdef CURL_ENABLED + return ctx->networking.enabled; +#else + return ctx->networking.enabled && + ctx->networking.open != NS_PROJ::no_op_network_open; +#endif +} + +// --------------------------------------------------------------------------- + +/** Define the URL endpoint to query for remote grids. +* +* This overrides the default endpoint in the PROJ configuration file or with +* the PROJ_NETWORK_ENDPOINT environment variable. +* +* @param ctx PROJ context, or NULL +* @param url Endpoint URL. Must NOT be NULL. +*/ +void proj_context_set_url_endpoint(PJ_CONTEXT *ctx, const char *url) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + // Load ini file, now so as to override its network settings + pj_load_ini(ctx); + ctx->endpoint = url; +} + +// --------------------------------------------------------------------------- + +//! @cond Doxygen_Suppress + +bool pj_context_is_network_enabled(PJ_CONTEXT *ctx) { + if (ctx == nullptr) { + ctx = pj_get_default_ctx(); + } + if (ctx->networking.enabled_env_variable_checked) { + return ctx->networking.enabled; + } + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled && enabled[0] != '\0') { + ctx->networking.enabled = ci_equal(enabled, "ON") || + ci_equal(enabled, "YES") || + ci_equal(enabled, "TRUE"); + } + pj_load_ini(ctx); + ctx->networking.enabled_env_variable_checked = true; + return ctx->networking.enabled; +} + +//! @endcond diff --git a/src/filemanager.hpp b/src/filemanager.hpp new file mode 100644 index 00000000..972634c2 --- /dev/null +++ b/src/filemanager.hpp @@ -0,0 +1,80 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: File manager + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#ifndef FILEMANAGER_HPP_INCLUDED +#define FILEMANAGER_HPP_INCLUDED + +#include <memory> + +#include "proj.h" +#include "proj/util.hpp" + +NS_PROJ_START + +//! @cond Doxygen_Suppress + +class File; + +class FileManager { + private: + FileManager() = delete; + + public: + // "Low-level" interface. + static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename); + + // "High-level" interface, honoring PROJ_LIB and the like. + static std::unique_ptr<File> open_resource_file(PJ_CONTEXT *ctx, + const char *name); + + static void fillDefaultNetworkInterface(PJ_CONTEXT *ctx); + + static void clearCache(); +}; + +// --------------------------------------------------------------------------- + +class File { + protected: + std::string name_; + explicit File(const std::string &name); + + public: + virtual ~File(); + virtual size_t read(void *buffer, size_t sizeBytes) = 0; + virtual bool seek(unsigned long long offset, int whence = SEEK_SET) = 0; + virtual unsigned long long tell() = 0; + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; + + const std::string &name() const { return name_; } +}; + +//! @endcond Doxygen_Suppress + +NS_PROJ_END + +#endif // FILEMANAGER_HPP_INCLUDED
\ No newline at end of file diff --git a/src/grids.cpp b/src/grids.cpp index 7d19b1f7..5a99106b 100644 --- a/src/grids.cpp +++ b/src/grids.cpp @@ -29,11 +29,21 @@ #ifndef FROM_PROJ_CPP #define FROM_PROJ_CPP #endif +#define LRU11_DO_NOT_DEFINE_OUT_OF_CLASS_METHODS #include "grids.hpp" +#include "filemanager.hpp" #include "proj/internal/internal.hpp" +#include "proj/internal/lru_cache.hpp" #include "proj_internal.h" +#ifdef TIFF_ENABLED +#include "tiffio.h" +#endif + +#include <algorithm> +#include <cmath> + NS_PROJ_START using namespace internal; @@ -64,14 +74,31 @@ static void swap_words(void *dataIn, size_t word_size, size_t word_count) } } +// --------------------------------------------------------------------------- + bool ExtentAndRes::fullWorldLongitude() const { return eastLon - westLon + resLon >= 2 * M_PI - 1e-10; } // --------------------------------------------------------------------------- -Grid::Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn) - : m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} +bool ExtentAndRes::contains(const ExtentAndRes &other) const { + return other.westLon >= westLon && other.eastLon <= eastLon && + other.southLat >= southLat && other.northLat <= northLat; +} + +// --------------------------------------------------------------------------- + +bool ExtentAndRes::intersects(const ExtentAndRes &other) const { + return other.westLon < eastLon && westLon <= other.westLon && + other.southLat < northLat && southLat <= other.northLat; +} + +// --------------------------------------------------------------------------- + +Grid::Grid(const std::string &name, int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : m_name(name), m_width(widthIn), m_height(heightIn), m_extent(extentIn) {} // --------------------------------------------------------------------------- @@ -79,9 +106,9 @@ Grid::~Grid() = default; // --------------------------------------------------------------------------- -VerticalShiftGrid::VerticalShiftGrid(int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} +VerticalShiftGrid::VerticalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -107,11 +134,12 @@ static ExtentAndRes globalExtent() { class NullVerticalShiftGrid : public VerticalShiftGrid { public: - NullVerticalShiftGrid() : VerticalShiftGrid(3, 3, globalExtent()) {} + NullVerticalShiftGrid() : VerticalShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } bool valueAt(int, int, float &out) const override; bool isNodata(float, double) const override { return false; } + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- @@ -125,38 +153,47 @@ bool NullVerticalShiftGrid::valueAt(int, int, float &out) const { class GTXVerticalShiftGrid : public VerticalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr<File> m_fp; GTXVerticalShiftGrid(const GTXVerticalShiftGrid &) = delete; GTXVerticalShiftGrid &operator=(const GTXVerticalShiftGrid &) = delete; public: - GTXVerticalShiftGrid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : VerticalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), m_fp(fp) { - } + explicit GTXVerticalShiftGrid(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp, + const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : VerticalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(std::move(fp)) {} ~GTXVerticalShiftGrid() override; bool valueAt(int x, int y, float &out) const override; bool isNodata(float val, double multiplier) const override; - static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, PAFile fp); + static GTXVerticalShiftGrid *open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &name); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- -GTXVerticalShiftGrid::~GTXVerticalShiftGrid() { pj_ctx_fclose(m_ctx, m_fp); } +GTXVerticalShiftGrid::~GTXVerticalShiftGrid() = default; // --------------------------------------------------------------------------- -GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp) { +GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, + std::unique_ptr<File> fp, + const std::string &name) { unsigned char header[40]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -206,7 +243,8 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp) { extent.eastLon = (xorigin + xstep * (columns - 1)) * DEG_TO_RAD; extent.northLat = (yorigin + ystep * (rows - 1)) * DEG_TO_RAD; - return new GTXVerticalShiftGrid(ctx, fp, columns, rows, extent); + return new GTXVerticalShiftGrid(ctx, std::move(fp), name, columns, rows, + extent); } // --------------------------------------------------------------------------- @@ -214,8 +252,8 @@ GTXVerticalShiftGrid *GTXVerticalShiftGrid::open(PJ_CONTEXT *ctx, PAFile fp) { bool GTXVerticalShiftGrid::valueAt(int x, int y, float &out) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); - pj_ctx_fseek(m_ctx, m_fp, 40 + sizeof(float) * (y * m_width + x), SEEK_SET); - if (pj_ctx_fread(m_ctx, &out, sizeof(out), 1, m_fp) != 1) { + m_fp->seek(40 + sizeof(float) * (y * m_width + x)); + if (m_fp->read(&out, sizeof(out)) != sizeof(out)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -246,6 +284,1026 @@ VerticalShiftGridSet::~VerticalShiftGridSet() = default; // --------------------------------------------------------------------------- +static bool IsTIFF(size_t header_size, const unsigned char *header) { + // Test combinations of signature for ClassicTIFF/BigTIFF little/big endian + return header_size >= 4 && (((header[0] == 'I' && header[1] == 'I') || + (header[0] == 'M' && header[1] == 'M')) && + ((header[2] == 0x2A && header[3] == 0) || + (header[3] == 0x2A && header[2] == 0) || + (header[2] == 0x2B && header[3] == 0) || + (header[3] == 0x2B && header[2] == 0))); +} + +#ifdef TIFF_ENABLED + +// --------------------------------------------------------------------------- + +enum class TIFFDataType { Int16, UInt16, Int32, UInt32, Float32, Float64 }; + +// --------------------------------------------------------------------------- + +constexpr uint16 TIFFTAG_GEOPIXELSCALE = 33550; +constexpr uint16 TIFFTAG_GEOTIEPOINTS = 33922; +constexpr uint16 TIFFTAG_GEOTRANSMATRIX = 34264; +constexpr uint16 TIFFTAG_GEOKEYDIRECTORY = 34735; +constexpr uint16 TIFFTAG_GEODOUBLEPARAMS = 34736; +constexpr uint16 TIFFTAG_GEOASCIIPARAMS = 34737; +constexpr uint16 TIFFTAG_GDAL_METADATA = 42112; +constexpr uint16 TIFFTAG_GDAL_NODATA = 42113; + +// --------------------------------------------------------------------------- + +class BlockCache { + public: + void insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector<unsigned char> &data); + std::shared_ptr<std::vector<unsigned char>> get(uint32 ifdIdx, + uint32 blockNumber); + + private: + struct Key { + uint32 ifdIdx; + uint32 blockNumber; + + Key(uint32 ifdIdxIn, uint32 blockNumberIn) + : ifdIdx(ifdIdxIn), blockNumber(blockNumberIn) {} + bool operator==(const Key &other) const { + return ifdIdx == other.ifdIdx && blockNumber == other.blockNumber; + } + }; + + struct KeyHasher { + std::size_t operator()(const Key &k) const { + return k.ifdIdx ^ (k.blockNumber << 16) ^ (k.blockNumber >> 16); + } + }; + + static constexpr int NUM_BLOCKS_AT_CROSSING_TILES = 4; + static constexpr int MAX_SAMPLE_COUNT = 3; + lru11::Cache< + Key, std::shared_ptr<std::vector<unsigned char>>, lru11::NullLock, + std::unordered_map< + Key, + typename std::list<lru11::KeyValuePair< + Key, std::shared_ptr<std::vector<unsigned char>>>>::iterator, + KeyHasher>> + cache_{NUM_BLOCKS_AT_CROSSING_TILES * MAX_SAMPLE_COUNT}; +}; + +// --------------------------------------------------------------------------- + +void BlockCache::insert(uint32 ifdIdx, uint32 blockNumber, + const std::vector<unsigned char> &data) { + cache_.insert(Key(ifdIdx, blockNumber), + std::make_shared<std::vector<unsigned char>>(data)); +} + +// --------------------------------------------------------------------------- + +std::shared_ptr<std::vector<unsigned char>> +BlockCache::get(uint32 ifdIdx, uint32 blockNumber) { + std::shared_ptr<std::vector<unsigned char>> ret; + cache_.tryGet(Key(ifdIdx, blockNumber), ret); + return ret; +} + +// --------------------------------------------------------------------------- + +class GTiffGrid : public Grid { + PJ_CONTEXT *m_ctx; // owned by the belonging GTiffDataset + TIFF *m_hTIFF; // owned by the belonging GTiffDataset + BlockCache &m_cache; // owned by the belonging GTiffDataset + uint32 m_ifdIdx; + TIFFDataType m_dt; + uint16 m_samplesPerPixel; + uint16 m_planarConfig; + bool m_bottomUp; + toff_t m_dirOffset; + bool m_tiled; + uint32 m_blockWidth = 0; + uint32 m_blockHeight = 0; + mutable std::vector<unsigned char> m_buffer{}; + unsigned m_blocksPerRow = 0; + unsigned m_blocksPerCol = 0; + std::map<int, double> m_mapOffset{}; + std::map<int, double> m_mapScale{}; + std::map<std::pair<int, std::string>, std::string> m_metadata{}; + bool m_hasNodata = false; + float m_noData = 0.0f; + uint32 m_subfileType = 0; + + GTiffGrid(const GTiffGrid &) = delete; + GTiffGrid &operator=(const GTiffGrid &) = delete; + + void getScaleOffset(double &scale, double &offset, uint16 sample) const; + + template <class T> + float readValue(const std::vector<unsigned char> &buffer, + uint32 offsetInBlock, uint16 sample) const; + + public: + GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, uint32 ifdIdx, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn, TIFFDataType dtIn, + uint16 samplesPerPixelIn, uint16 planarConfig, bool bottomUpIn); + + ~GTiffGrid() override; + + uint16 samplesPerPixel() const { return m_samplesPerPixel; } + + bool valueAt(uint16 sample, int x, int y, float &out) const; + + bool isNodata(float val) const; + + std::string metadataItem(const std::string &key, int sample = -1) const; + + uint32 subfileType() const { return m_subfileType; } + + void reassign_context(PJ_CONTEXT *ctx) { m_ctx = ctx; } +}; + +// --------------------------------------------------------------------------- + +GTiffGrid::GTiffGrid(PJ_CONTEXT *ctx, TIFF *hTIFF, BlockCache &cache, + uint32 ifdIdx, const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn, + TIFFDataType dtIn, uint16 samplesPerPixelIn, + uint16 planarConfig, bool bottomUpIn) + : Grid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), m_hTIFF(hTIFF), + m_cache(cache), m_ifdIdx(ifdIdx), m_dt(dtIn), + m_samplesPerPixel(samplesPerPixelIn), m_planarConfig(planarConfig), + m_bottomUp(bottomUpIn), m_dirOffset(TIFFCurrentDirOffset(hTIFF)), + m_tiled(TIFFIsTiled(hTIFF) != 0) { + + if (m_tiled) { + TIFFGetField(m_hTIFF, TIFFTAG_TILEWIDTH, &m_blockWidth); + TIFFGetField(m_hTIFF, TIFFTAG_TILELENGTH, &m_blockHeight); + } else { + m_blockWidth = widthIn; + TIFFGetField(m_hTIFF, TIFFTAG_ROWSPERSTRIP, &m_blockHeight); + if (m_blockHeight > static_cast<unsigned>(m_height)) + m_blockHeight = m_height; + } + + TIFFGetField(m_hTIFF, TIFFTAG_SUBFILETYPE, &m_subfileType); + + m_blocksPerRow = (m_width + m_blockWidth - 1) / m_blockWidth; + m_blocksPerCol = (m_height + m_blockHeight - 1) / m_blockHeight; + + const char *text = nullptr; + // Poor-man XML parsing of TIFFTAG_GDAL_METADATA tag. Hopefully good + // enough for our purposes. + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_METADATA, &text)) { + const char *ptr = text; + while (true) { + ptr = strstr(ptr, "<Item "); + if (ptr == nullptr) + break; + const char *endTag = strchr(ptr, '>'); + if (endTag == nullptr) + break; + const char *endValue = strchr(endTag, '<'); + if (endValue == nullptr) + break; + + std::string tag; + tag.append(ptr, endTag - ptr); + + std::string value; + value.append(endTag + 1, endValue - (endTag + 1)); + + std::string name; + auto namePos = tag.find("name=\""); + if (namePos == std::string::npos) + break; + { + namePos += strlen("name=\""); + const auto endQuote = tag.find('"', namePos); + if (endQuote == std::string::npos) + break; + name = tag.substr(namePos, endQuote - namePos); + } + + const auto samplePos = tag.find("sample=\""); + int sample = -1; + if (samplePos != std::string::npos) { + sample = atoi(tag.c_str() + samplePos + strlen("sample=\"")); + } + + m_metadata[std::pair<int, std::string>(sample, name)] = value; + + auto rolePos = tag.find("role=\""); + if (rolePos != std::string::npos) { + rolePos += strlen("role=\""); + const auto endQuote = tag.find('"', rolePos); + if (endQuote == std::string::npos) + break; + const auto role = tag.substr(rolePos, endQuote - rolePos); + if (role == "offset") { + if (sample >= 0) { + try { + m_mapOffset[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } else if (role == "scale") { + if (sample >= 0) { + try { + m_mapScale[sample] = c_locale_stod(value); + } catch (const std::exception &) { + } + } + } + } + + ptr = endValue + 1; + } + } + + if (TIFFGetField(m_hTIFF, TIFFTAG_GDAL_NODATA, &text)) { + try { + m_noData = static_cast<float>(c_locale_stod(text)); + m_hasNodata = true; + } catch (const std::exception &) { + } + } + + auto oIter = m_metadata.find(std::pair<int, std::string>(-1, "grid_name")); + if (oIter != m_metadata.end()) { + m_name += ", " + oIter->second; + } +} + +// --------------------------------------------------------------------------- + +GTiffGrid::~GTiffGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffGrid::getScaleOffset(double &scale, double &offset, + uint16 sample) const { + { + auto iter = m_mapScale.find(sample); + if (iter != m_mapScale.end()) + scale = iter->second; + } + + { + auto iter = m_mapOffset.find(sample); + if (iter != m_mapOffset.end()) + offset = iter->second; + } +} + +// --------------------------------------------------------------------------- + +template <class T> +float GTiffGrid::readValue(const std::vector<unsigned char> &buffer, + uint32 offsetInBlock, uint16 sample) const { + const auto ptr = reinterpret_cast<const T *>(buffer.data()); + assert(offsetInBlock < buffer.size() / sizeof(T)); + const auto val = ptr[offsetInBlock]; + if (!m_hasNodata || static_cast<float>(val) != m_noData) { + double scale = 1; + double offset = 0; + getScaleOffset(scale, offset, sample); + return static_cast<float>(val * scale + offset); + } else { + return static_cast<float>(val); + } +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::valueAt(uint16 sample, int x, int yFromBottom, + float &out) const { + assert(x >= 0 && yFromBottom >= 0 && x < m_width && yFromBottom < m_height); + assert(sample < m_samplesPerPixel); + + const int blockX = x / m_blockWidth; + + // All non-TIFF grids have the first rows in the file being the one + // corresponding to the southern-most row. In GeoTIFF, the convention is + // *generally* different (when m_bottomUp == false), TIFF being an + // image-oriented image. If m_bottomUp == true, then we had GeoTIFF hints + // that the first row of the image is the southern-most. + const int yTIFF = m_bottomUp ? yFromBottom : m_height - 1 - yFromBottom; + const int blockY = yTIFF / m_blockHeight; + + uint32 blockId = blockY * m_blocksPerRow + blockX; + if (m_planarConfig == PLANARCONFIG_SEPARATE) { + blockId += sample * m_blocksPerCol * m_blocksPerRow; + } + + auto cachedBuffer = m_cache.get(m_ifdIdx, blockId); + std::vector<unsigned char> *pBuffer = &m_buffer; + if (cachedBuffer != nullptr) { + // Safe as we don't access the cache before pBuffer is used + pBuffer = cachedBuffer.get(); + } else { + if (TIFFCurrentDirOffset(m_hTIFF) != m_dirOffset && + !TIFFSetSubDirectory(m_hTIFF, m_dirOffset)) { + return false; + } + if (m_buffer.empty()) { + const auto blockSize = static_cast<size_t>( + m_tiled ? TIFFTileSize64(m_hTIFF) : TIFFStripSize64(m_hTIFF)); + try { + m_buffer.resize(blockSize); + } catch (const std::exception &e) { + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + return false; + } + } + + if (m_tiled) { + if (TIFFReadEncodedTile(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } else { + if (TIFFReadEncodedStrip(m_hTIFF, blockId, m_buffer.data(), + m_buffer.size()) == -1) { + return false; + } + } + + try { + m_cache.insert(m_ifdIdx, blockId, m_buffer); + } catch (const std::exception &e) { + // Should normally not happen + pj_log(m_ctx, PJ_LOG_ERROR, "Exception %s", e.what()); + } + } + + uint32 offsetInBlock = + (x % m_blockWidth) + (yTIFF % m_blockHeight) * m_blockWidth; + if (m_planarConfig == PLANARCONFIG_CONTIG) + offsetInBlock = offsetInBlock * m_samplesPerPixel + sample; + + switch (m_dt) { + case TIFFDataType::Int16: + out = readValue<short>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::UInt16: + out = readValue<unsigned short>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::Int32: + out = readValue<int>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::UInt32: + out = readValue<unsigned int>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::Float32: + out = readValue<float>(*pBuffer, offsetInBlock, sample); + break; + + case TIFFDataType::Float64: + out = readValue<double>(*pBuffer, offsetInBlock, sample); + break; + } + + return true; +} + +// --------------------------------------------------------------------------- + +bool GTiffGrid::isNodata(float val) const { + return (m_hasNodata && val == m_noData) || std::isnan(val); +} + +// --------------------------------------------------------------------------- + +std::string GTiffGrid::metadataItem(const std::string &key, int sample) const { + auto iter = m_metadata.find(std::pair<int, std::string>(sample, key)); + if (iter == m_metadata.end()) { + return std::string(); + } + return iter->second; +} + +// --------------------------------------------------------------------------- + +class GTiffDataset { + PJ_CONTEXT *m_ctx; + std::unique_ptr<File> m_fp; + TIFF *m_hTIFF = nullptr; + bool m_hasNextGrid = false; + uint32 m_ifdIdx = 0; + toff_t m_nextDirOffset = 0; + std::string m_filename{}; + BlockCache m_cache{}; + + GTiffDataset(const GTiffDataset &) = delete; + GTiffDataset &operator=(const GTiffDataset &) = delete; + + // libtiff I/O routines + static tsize_t tiffReadProc(thandle_t fd, tdata_t buf, tsize_t size) { + GTiffDataset *self = static_cast<GTiffDataset *>(fd); + return self->m_fp->read(buf, size); + } + + static tsize_t tiffWriteProc(thandle_t, tdata_t, tsize_t) { + assert(false); + return 0; + } + + static toff_t tiffSeekProc(thandle_t fd, toff_t off, int whence) { + GTiffDataset *self = static_cast<GTiffDataset *>(fd); + if (self->m_fp->seek(off, whence)) + return static_cast<toff_t>(self->m_fp->tell()); + else + return static_cast<toff_t>(-1); + } + + static int tiffCloseProc(thandle_t) { + // done in destructor + return 0; + } + + static toff_t tiffSizeProc(thandle_t fd) { + GTiffDataset *self = static_cast<GTiffDataset *>(fd); + const auto old_off = self->m_fp->tell(); + self->m_fp->seek(0, SEEK_END); + const auto file_size = static_cast<toff_t>(self->m_fp->tell()); + self->m_fp->seek(old_off); + return file_size; + } + + static int tiffMapProc(thandle_t, tdata_t *, toff_t *) { return (0); } + + static void tiffUnmapProc(thandle_t, tdata_t, toff_t) {} + + public: + GTiffDataset(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : m_ctx(ctx), m_fp(std::move(fp)) {} + virtual ~GTiffDataset(); + + bool openTIFF(const std::string &filename); + + std::unique_ptr<GTiffGrid> nextGrid(); + + void reassign_context(PJ_CONTEXT *ctx) { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +GTiffDataset::~GTiffDataset() { + if (m_hTIFF) + TIFFClose(m_hTIFF); +} + +// --------------------------------------------------------------------------- +class OneTimeTIFFTagInit { + + static TIFFExtendProc ParentExtender; + + // Function called by libtiff when initializing a TIFF directory + static void GTiffTagExtender(TIFF *tif) { + static const TIFFFieldInfo xtiffFieldInfo[] = { + // GeoTIFF tags + {TIFFTAG_GEOPIXELSCALE, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoPixelScale")}, + {TIFFTAG_GEOTIEPOINTS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoTiePoints")}, + {TIFFTAG_GEOTRANSMATRIX, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoTransformationMatrix")}, + + {TIFFTAG_GEOKEYDIRECTORY, -1, -1, TIFF_SHORT, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoKeyDirectory")}, + {TIFFTAG_GEODOUBLEPARAMS, -1, -1, TIFF_DOUBLE, FIELD_CUSTOM, TRUE, + TRUE, const_cast<char *>("GeoDoubleParams")}, + {TIFFTAG_GEOASCIIPARAMS, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast<char *>("GeoASCIIParams")}, + + // GDAL tags + {TIFFTAG_GDAL_METADATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, + FALSE, const_cast<char *>("GDALMetadata")}, + {TIFFTAG_GDAL_NODATA, -1, -1, TIFF_ASCII, FIELD_CUSTOM, TRUE, FALSE, + const_cast<char *>("GDALNoDataValue")}, + + }; + + if (ParentExtender) + (*ParentExtender)(tif); + + TIFFMergeFieldInfo(tif, xtiffFieldInfo, + sizeof(xtiffFieldInfo) / sizeof(xtiffFieldInfo[0])); + } + + public: + OneTimeTIFFTagInit() { + assert(ParentExtender == nullptr); + // Install our TIFF tag extender + ParentExtender = TIFFSetTagExtender(GTiffTagExtender); + } +}; + +TIFFExtendProc OneTimeTIFFTagInit::ParentExtender = nullptr; + +// --------------------------------------------------------------------------- + +bool GTiffDataset::openTIFF(const std::string &filename) { + static OneTimeTIFFTagInit oneTimeTIFFTagInit; + m_hTIFF = + TIFFClientOpen(filename.c_str(), "r", static_cast<thandle_t>(this), + GTiffDataset::tiffReadProc, GTiffDataset::tiffWriteProc, + GTiffDataset::tiffSeekProc, GTiffDataset::tiffCloseProc, + GTiffDataset::tiffSizeProc, GTiffDataset::tiffMapProc, + GTiffDataset::tiffUnmapProc); + + m_filename = filename; + m_hasNextGrid = true; + return m_hTIFF != nullptr; +} +// --------------------------------------------------------------------------- + +std::unique_ptr<GTiffGrid> GTiffDataset::nextGrid() { + if (!m_hasNextGrid) + return nullptr; + if (m_nextDirOffset) { + TIFFSetSubDirectory(m_hTIFF, m_nextDirOffset); + } + + uint32 width = 0; + uint32 height = 0; + TIFFGetField(m_hTIFF, TIFFTAG_IMAGEWIDTH, &width); + TIFFGetField(m_hTIFF, TIFFTAG_IMAGELENGTH, &height); + if (width == 0 || height == 0 || width > INT_MAX || height > INT_MAX) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid image size"); + return nullptr; + } + + uint16 samplesPerPixel = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLESPERPIXEL, &samplesPerPixel)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SamplesPerPixel tag"); + return nullptr; + } + if (samplesPerPixel == 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Invalid SamplesPerPixel value"); + return nullptr; + } + + uint16 bitsPerSample = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_BITSPERSAMPLE, &bitsPerSample)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing BitsPerSample tag"); + return nullptr; + } + + uint16 planarConfig = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PLANARCONFIG, &planarConfig)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing PlanarConfig tag"); + return nullptr; + } + + uint16 sampleFormat = 0; + if (!TIFFGetField(m_hTIFF, TIFFTAG_SAMPLEFORMAT, &sampleFormat)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Missing SampleFormat tag"); + return nullptr; + } + + TIFFDataType dt; + if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 16) + dt = TIFFDataType::Int16; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 16) + dt = TIFFDataType::UInt16; + else if (sampleFormat == SAMPLEFORMAT_INT && bitsPerSample == 32) + dt = TIFFDataType::Int32; + else if (sampleFormat == SAMPLEFORMAT_UINT && bitsPerSample == 32) + dt = TIFFDataType::UInt32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 32) + dt = TIFFDataType::Float32; + else if (sampleFormat == SAMPLEFORMAT_IEEEFP && bitsPerSample == 64) + dt = TIFFDataType::Float64; + else { + pj_log( + m_ctx, PJ_LOG_ERROR, + "Unsupported combination of SampleFormat and BitsPerSample values"); + return nullptr; + } + + uint16 photometric = PHOTOMETRIC_MINISBLACK; + if (!TIFFGetField(m_hTIFF, TIFFTAG_PHOTOMETRIC, &photometric)) + photometric = PHOTOMETRIC_MINISBLACK; + if (photometric != PHOTOMETRIC_MINISBLACK) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported Photometric value"); + return nullptr; + } + + uint16 compression = COMPRESSION_NONE; + if (!TIFFGetField(m_hTIFF, TIFFTAG_COMPRESSION, &compression)) + compression = COMPRESSION_NONE; + + if (compression != COMPRESSION_NONE && + !TIFFIsCODECConfigured(compression)) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Cannot open TIFF file due to missing codec."); + return nullptr; + } + // We really don't want to try dealing with old-JPEG images + if (compression == COMPRESSION_OJPEG) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported compression method."); + return nullptr; + } + + const auto blockSize = TIFFIsTiled(m_hTIFF) ? TIFFTileSize64(m_hTIFF) + : TIFFStripSize64(m_hTIFF); + if (blockSize == 0 || blockSize > 64 * 1024 * 2014) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported block size."); + return nullptr; + } + + unsigned short count = 0; + unsigned short *geokeys = nullptr; + bool pixelIsArea = false; + if (!TIFFGetField(m_hTIFF, TIFFTAG_GEOKEYDIRECTORY, &count, &geokeys)) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, "No GeoKeys tag"); + } else { + if (count < 4 || (count % 4) != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoKeys tag"); + return nullptr; + } + + if (geokeys[0] != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "Unsupported GeoTIFF major version"); + return nullptr; + } + // We only know that we support GeoTIFF 1.0 and 1.1 at that time + if (geokeys[1] != 1 || geokeys[2] > 1) { + pj_log(m_ctx, PJ_LOG_DEBUG_MINOR, + "GeoTIFF %d.%d possibly not handled", geokeys[1], + geokeys[2]); + } + + for (unsigned int i = 4; i + 3 < count; i += 4) { + constexpr unsigned short GTModelTypeGeoKey = 1024; + constexpr unsigned short ModelTypeGeographic = 2; + + constexpr unsigned short GTRasterTypeGeoKey = 1025; + constexpr unsigned short RasterPixelIsArea = 1; + // constexpr unsigned short RasterPixelIsPoint = 2; + + if (geokeys[i] == GTModelTypeGeoKey) { + if (geokeys[i + 3] != ModelTypeGeographic) { + pj_log(m_ctx, PJ_LOG_ERROR, "Only GTModelTypeGeoKey = " + "ModelTypeGeographic is " + "supported"); + return nullptr; + } + } else if (geokeys[i] == GTRasterTypeGeoKey) { + if (geokeys[i + 3] == RasterPixelIsArea) { + pixelIsArea = true; + } + } + } + } + + double hRes = 0; + double vRes = 0; + double westLon = 0; + double northLat = 0; + + double *matrix = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTRANSMATRIX, &count, &matrix) && + count == 16) { + // If using GDAL to produce a bottom-up georeferencing, it will produce + // a GeoTransformationMatrix, since negative values in GeoPixelScale + // have historically been implementation bugs. + if (matrix[1] != 0 || matrix[4] != 0) { + pj_log(m_ctx, PJ_LOG_ERROR, "Rotational terms not supported in " + "GeoTransformationMatrix tag"); + return nullptr; + } + + westLon = matrix[3]; + hRes = matrix[0]; + northLat = matrix[7]; + vRes = -matrix[5]; // negation to simulate GeoPixelScale convention + } else { + double *geopixelscale = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOPIXELSCALE, &count, + &geopixelscale) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoPixelScale tag"); + return nullptr; + } + if (count != 3) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoPixelScale tag"); + return nullptr; + } + hRes = geopixelscale[0]; + vRes = geopixelscale[1]; + + double *geotiepoints = nullptr; + if (TIFFGetField(m_hTIFF, TIFFTAG_GEOTIEPOINTS, &count, + &geotiepoints) != 1) { + pj_log(m_ctx, PJ_LOG_ERROR, "No GeoTiePoints tag"); + return nullptr; + } + if (count != 6) { + pj_log(m_ctx, PJ_LOG_ERROR, + "Wrong number of values in GeoTiePoints tag"); + return nullptr; + } + + westLon = geotiepoints[3] - geotiepoints[0] * hRes; + northLat = geotiepoints[4] + geotiepoints[1] * vRes; + } + + if (pixelIsArea) { + westLon += 0.5 * hRes; + northLat -= 0.5 * vRes; + } + + ExtentAndRes extent; + extent.westLon = westLon * DEG_TO_RAD; + extent.northLat = northLat * DEG_TO_RAD; + extent.resLon = hRes * DEG_TO_RAD; + extent.resLat = fabs(vRes) * DEG_TO_RAD; + extent.eastLon = (westLon + hRes * (width - 1)) * DEG_TO_RAD; + extent.southLat = (northLat - vRes * (height - 1)) * DEG_TO_RAD; + + if (vRes < 0) { + std::swap(extent.northLat, extent.southLat); + } + + if (!(fabs(extent.westLon) <= 4 * M_PI && + fabs(extent.eastLon) <= 4 * M_PI && + fabs(extent.northLat) <= M_PI + 1e-5 && + fabs(extent.southLat) <= M_PI + 1e-5 && + extent.westLon < extent.eastLon && + extent.southLat < extent.northLat && extent.resLon > 1e-10 && + extent.resLat > 1e-10)) { + pj_log(m_ctx, PJ_LOG_ERROR, "Inconsistent georeferencing for %s", + m_filename.c_str()); + return nullptr; + } + + auto ret = std::unique_ptr<GTiffGrid>(new GTiffGrid( + m_ctx, m_hTIFF, m_cache, m_ifdIdx, m_filename, width, height, extent, + dt, samplesPerPixel, planarConfig, vRes < 0)); + m_ifdIdx++; + m_hasNextGrid = TIFFReadDirectory(m_hTIFF) != 0; + m_nextDirOffset = TIFFCurrentDirOffset(m_hTIFF); + return ret; +} + +// --------------------------------------------------------------------------- + +class GTiffVGridShiftSet : public VerticalShiftGridSet, public GTiffDataset { + + GTiffVGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : GTiffDataset(ctx, std::move(fp)) {} + + public: + ~GTiffVGridShiftSet() override; + + static std::unique_ptr<GTiffVGridShiftSet> + open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + VerticalShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } +}; + +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +template <class GridType, class GenericGridType> +static void +insertIntoHierarchy(PJ_CONTEXT *ctx, std::unique_ptr<GridType> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<GenericGridType>> &topGrids, + std::map<std::string, GridType *> &mapGrids) { + const auto &extent = grid->extentAndRes(); + + // If we have one or both of grid_name and parent_grid_name, try to use + // the names to recreate the hiearchy + if (!gridName.empty()) { + if (mapGrids.find(gridName) != mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Several grids called %s found!", + gridName.c_str()); + } + mapGrids[gridName] = grid.get(); + } + bool gridInserted = false; + if (!parentName.empty()) { + auto iter = mapGrids.find(parentName); + if (iter == mapGrids.end()) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to non-existing parent %s. " + "Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } else { + if (iter->second->extentAndRes().contains(extent)) { + iter->second->m_children.emplace_back(std::move(grid)); + gridInserted = true; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Grid %s refers to parent %s, but its extent is " + "not included in it. Using bounding-box method.", + gridName.c_str(), parentName.c_str()); + } + } + } else if (!gridName.empty()) { + topGrids.emplace_back(std::move(grid)); + gridInserted = true; + } + + // Fallback to analyzing spatial extents + if (!gridInserted) { + for (const auto &candidateParent : topGrids) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GridType *>(candidateParent.get()) + ->insertGrid(ctx, std::move(grid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + topGrids.emplace_back(std::move(grid)); + } + } +} + +#ifdef TIFF_ENABLED +// --------------------------------------------------------------------------- + +class GTiffVGrid : public VerticalShiftGrid { + friend void insertIntoHierarchy<GTiffVGrid, VerticalShiftGrid>( + PJ_CONTEXT *ctx, std::unique_ptr<GTiffVGrid> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<VerticalShiftGrid>> &topGrids, + std::map<std::string, GTiffVGrid *> &mapGrids); + + std::unique_ptr<GTiffGrid> m_grid; + uint16 m_idxSample; + + public: + GTiffVGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxSample); + + ~GTiffVGrid() override; + + bool valueAt(int x, int y, float &out) const override { + return m_grid->valueAt(m_idxSample, x, y, out); + } + + bool isNodata(float val, double /* multiplier */) const override { + return m_grid->isNodata(val); + } + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr<GTiffVGrid> &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +GTiffVGridShiftSet::~GTiffVGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffVGrid::GTiffVGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxSample) + : VerticalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), + m_grid(std::move(grid)), m_idxSample(idxSample) {} + +// --------------------------------------------------------------------------- + +GTiffVGrid::~GTiffVGrid() = default; + +// --------------------------------------------------------------------------- + +void GTiffVGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffVGrid> &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GTiffVGrid *>(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<GTiffVGridShiftSet> +GTiffVGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + auto set = std::unique_ptr<GTiffVGridShiftSet>( + new GTiffVGridShiftSet(ctx, std::move(fp))); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + uint16 idxSample = 0; + + std::map<std::string, GTiffVGrid *> mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + // Identify the index of the geoid_undulation/vertical_offset + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForShift = false; + for (int i = 0; i < static_cast<int>(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "geoid_undulation" || desc == "vertical_offset") { + idxSample = static_cast<uint16>(i); + foundDescriptionForShift = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForShift) { + if (ifd > 0) { + // Assuming that extra IFD without our channel of interest + // can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "geoid_undulation/vertical_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "geoid_undulation/vertical_offset channel"); + return nullptr; + } + } + } + + if (idxSample >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); + + auto vgrid = + internal::make_unique<GTiffVGrid>(std::move(grid), idxSample); + + insertIntoHierarchy(ctx, std::move(vgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + std::unique_ptr<VerticalShiftGridSet> VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (filename == "null") { @@ -258,27 +1316,49 @@ VerticalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } - if (ends_with(filename, "gtx") || ends_with(filename, "GTX")) { - auto grid = GTXVerticalShiftGrid::open(ctx, fp); + const auto actualName(fp->name()); + if (ends_with(actualName, "gtx") || ends_with(actualName, "GTX")) { + auto grid = GTXVerticalShiftGrid::open(ctx, std::move(fp), actualName); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = std::unique_ptr<VerticalShiftGridSet>(new VerticalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "gtx"; set->m_grids.push_back(std::unique_ptr<VerticalShiftGrid>(grid)); return set; } + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { + return nullptr; + } + fp->seek(0); + + if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED + auto set = GTiffVGridShiftSet::open(ctx, std::move(fp), actualName); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; +#else + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); + return nullptr; +#endif + } + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized vertical grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } @@ -316,9 +1396,18 @@ const VerticalShiftGrid *VerticalShiftGridSet::gridAt(double lon, // --------------------------------------------------------------------------- -HorizontalShiftGrid::HorizontalShiftGrid(int widthIn, int heightIn, +void VerticalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + +HorizontalShiftGrid::HorizontalShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) - : Grid(widthIn, heightIn, extentIn) {} + : Grid(nameIn, widthIn, heightIn, extentIn) {} // --------------------------------------------------------------------------- @@ -337,16 +1426,20 @@ HorizontalShiftGridSet::~HorizontalShiftGridSet() = default; class NullHorizontalShiftGrid : public HorizontalShiftGrid { public: - NullHorizontalShiftGrid() : HorizontalShiftGrid(3, 3, globalExtent()) {} + NullHorizontalShiftGrid() + : HorizontalShiftGrid("null", 3, 3, globalExtent()) {} bool isNullGrid() const override { return true; } - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; + + void reassign_context(PJ_CONTEXT *) override {} }; // --------------------------------------------------------------------------- -bool NullHorizontalShiftGrid::valueAt(int, int, float &lonShift, +bool NullHorizontalShiftGrid::valueAt(int, int, bool, float &lonShift, float &latShift) const { lonShift = 0.0f; latShift = 0.0f; @@ -365,39 +1458,46 @@ static double to_double(const void *data) { class NTv1Grid : public HorizontalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr<File> m_fp; NTv1Grid(const NTv1Grid &) = delete; NTv1Grid &operator=(const NTv1Grid &) = delete; public: - NTv1Grid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, - const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + explicit NTv1Grid(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp, + const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn) + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(std::move(fp)) {} ~NTv1Grid() override; - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; - static NTv1Grid *open(PJ_CONTEXT *ctx, PAFile fp, + static NTv1Grid *open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- -NTv1Grid::~NTv1Grid() { pj_ctx_fclose(m_ctx, m_fp); } +NTv1Grid::~NTv1Grid() = default; // --------------------------------------------------------------------------- -NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, +NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, const std::string &filename) { unsigned char header[192]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -446,21 +1546,20 @@ NTv1Grid *NTv1Grid::open(PJ_CONTEXT *ctx, PAFile fp, const int rows = static_cast<int>( fabs((extent.northLat - extent.southLat) / extent.resLat + 0.5) + 1); - return new NTv1Grid(ctx, fp, columns, rows, extent); + return new NTv1Grid(ctx, std::move(fp), filename, columns, rows, extent); } // --------------------------------------------------------------------------- -bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool NTv1Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); double two_doubles[2]; // NTv1 is organized from east to west ! - pj_ctx_fseek(m_ctx, m_fp, - 192 + 2 * sizeof(double) * (y * m_width + m_width - 1 - x), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_doubles[0], sizeof(two_doubles), 1, m_fp) != - 1) { + m_fp->seek(192 + 2 * sizeof(double) * (y * m_width + m_width - 1 - x)); + if (m_fp->read(&two_doubles[0], sizeof(two_doubles)) != + sizeof(two_doubles)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -469,7 +1568,9 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { } /* convert seconds to radians */ latShift = static_cast<float>(two_doubles[0] * ((M_PI / 180.0) / 3600.0)); - lonShift = static_cast<float>(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast<float>(two_doubles[1] * ((M_PI / 180.0) / 3600.0)); return true; } @@ -478,39 +1579,46 @@ bool NTv1Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { class CTable2Grid : public HorizontalShiftGrid { PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr<File> m_fp; CTable2Grid(const CTable2Grid &) = delete; CTable2Grid &operator=(const CTable2Grid &) = delete; public: - CTable2Grid(PJ_CONTEXT *ctx, PAFile fp, int widthIn, int heightIn, + CTable2Grid(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_ctx(ctx), - m_fp(fp) {} + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), m_ctx(ctx), + m_fp(std::move(fp)) {} ~CTable2Grid() override; - bool valueAt(int, int, float &lonShift, float &latShift) const override; + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; - static CTable2Grid *open(PJ_CONTEXT *ctx, PAFile fp, + static CTable2Grid *open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- -CTable2Grid::~CTable2Grid() { pj_ctx_fclose(m_ctx, m_fp); } +CTable2Grid::~CTable2Grid() = default; // --------------------------------------------------------------------------- -CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, +CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, const std::string &filename) { unsigned char header[160]; /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fp->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -551,19 +1659,18 @@ CTable2Grid *CTable2Grid::open(PJ_CONTEXT *ctx, PAFile fp, extent.eastLon = extent.westLon + (width - 1) * extent.resLon; extent.northLat = extent.southLat + (height - 1) * extent.resLon; - return new CTable2Grid(ctx, fp, width, height, extent); + return new CTable2Grid(ctx, std::move(fp), filename, width, height, extent); } // --------------------------------------------------------------------------- -bool CTable2Grid::valueAt(int x, int y, float &lonShift, - float &latShift) const { +bool CTable2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_floats[2]; - pj_ctx_fseek(m_ctx, m_fp, 160 + 2 * sizeof(float) * (y * m_width + x), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_floats[0], sizeof(two_floats), 1, m_fp) != 1) { + m_fp->seek(160 + 2 * sizeof(float) * (y * m_width + x)); + if (m_fp->read(&two_floats[0], sizeof(two_floats)) != sizeof(two_floats)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -572,7 +1679,8 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, } latShift = two_floats[1]; - lonShift = two_floats[0]; + // west longitude positive convention ! + lonShift = (compensateNTConvention ? -1 : 1) * two_floats[0]; return true; } @@ -580,19 +1688,24 @@ bool CTable2Grid::valueAt(int x, int y, float &lonShift, // --------------------------------------------------------------------------- class NTv2GridSet : public HorizontalShiftGridSet { - PJ_CONTEXT *m_ctx; - PAFile m_fp; + std::unique_ptr<File> m_fp; NTv2GridSet(const NTv2GridSet &) = delete; NTv2GridSet &operator=(const NTv2GridSet &) = delete; - NTv2GridSet(PJ_CONTEXT *ctx, PAFile fp) : m_ctx(ctx), m_fp(fp) {} + explicit NTv2GridSet(std::unique_ptr<File> &&fp) : m_fp(std::move(fp)) {} public: ~NTv2GridSet() override; - static std::unique_ptr<NTv2GridSet> open(PJ_CONTEXT *ctx, PAFile fp, + static std::unique_ptr<NTv2GridSet> open(PJ_CONTEXT *ctx, + std::unique_ptr<File> fp, const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- @@ -602,7 +1715,7 @@ class NTv2Grid : public HorizontalShiftGrid { std::string m_name; PJ_CONTEXT *m_ctx; // owned by the parent NTv2GridSet - PAFile m_fp; // owned by the parent NTv2GridSet + File *m_fp; // owned by the parent NTv2GridSet unsigned long long m_offset; bool m_mustSwap; @@ -610,30 +1723,36 @@ class NTv2Grid : public HorizontalShiftGrid { NTv2Grid &operator=(const NTv2Grid &) = delete; public: - NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, PAFile fp, + NTv2Grid(const std::string &nameIn, PJ_CONTEXT *ctx, File *fp, unsigned long long offsetIn, bool mustSwapIn, int widthIn, int heightIn, const ExtentAndRes &extentIn) - : HorizontalShiftGrid(widthIn, heightIn, extentIn), m_name(nameIn), - m_ctx(ctx), m_fp(fp), m_offset(offsetIn), m_mustSwap(mustSwapIn) {} + : HorizontalShiftGrid(nameIn, widthIn, heightIn, extentIn), + m_name(nameIn), m_ctx(ctx), m_fp(fp), m_offset(offsetIn), + m_mustSwap(mustSwapIn) {} + + bool valueAt(int, int, bool, float &lonShift, + float &latShift) const override; - bool valueAt(int, int, float &lonShift, float &latShift) const override; + void reassign_context(PJ_CONTEXT *ctx) override { + m_ctx = ctx; + m_fp->reassign_context(ctx); + } }; // --------------------------------------------------------------------------- -bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { +bool NTv2Grid::valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const { assert(x >= 0 && y >= 0 && x < m_width && y < m_height); float two_float[2]; // NTv2 is organized from east to west ! // there are 4 components: lat shift, lon shift, lat error, lon error - pj_ctx_fseek( - m_ctx, m_fp, - // FIXME when fseek support unsigned long long - static_cast<long>(m_offset + - 4 * sizeof(float) * (y * m_width + m_width - 1 - x)), - SEEK_SET); - if (pj_ctx_fread(m_ctx, &two_float[0], sizeof(two_float), 1, m_fp) != 1) { + m_fp->seek( + m_offset + + 4 * sizeof(float) * + (static_cast<unsigned long long>(y) * m_width + m_width - 1 - x)); + if (m_fp->read(&two_float[0], sizeof(two_float)) != sizeof(two_float)) { pj_ctx_set_errno(m_ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return false; } @@ -642,19 +1761,23 @@ bool NTv2Grid::valueAt(int x, int y, float &lonShift, float &latShift) const { } /* convert seconds to radians */ latShift = static_cast<float>(two_float[0] * ((M_PI / 180.0) / 3600.0)); - lonShift = static_cast<float>(two_float[1] * ((M_PI / 180.0) / 3600.0)); + // west longitude positive convention ! + lonShift = (compensateNTConvention ? -1 : 1) * + static_cast<float>(two_float[1] * ((M_PI / 180.0) / 3600.0)); return true; } // --------------------------------------------------------------------------- -NTv2GridSet::~NTv2GridSet() { pj_ctx_fclose(m_ctx, m_fp); } +NTv2GridSet::~NTv2GridSet() = default; // --------------------------------------------------------------------------- -std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, +std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx, + std::unique_ptr<File> fp, const std::string &filename) { - auto set = std::unique_ptr<NTv2GridSet>(new NTv2GridSet(ctx, fp)); + File *fpRaw = fp.get(); + auto set = std::unique_ptr<NTv2GridSet>(new NTv2GridSet(std::move(fp))); set->m_name = filename; set->m_format = "ntv2"; @@ -663,7 +1786,7 @@ std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, /* -------------------------------------------------------------------- */ /* Read the header. */ /* -------------------------------------------------------------------- */ - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -692,7 +1815,7 @@ std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, /* ==================================================================== */ for (unsigned subfile = 0; subfile < num_subfiles; subfile++) { // Read header - if (pj_ctx_fread(ctx, header, sizeof(header), 1, fp) != 1) { + if (fpRaw->read(header, sizeof(header)) != sizeof(header)) { pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); return nullptr; } @@ -757,9 +1880,10 @@ std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, return nullptr; } - auto offset = pj_ctx_ftell(ctx, fp); - auto grid = std::unique_ptr<NTv2Grid>(new NTv2Grid( - gridName, ctx, fp, offset, must_swap, columns, rows, extent)); + const auto offset = fpRaw->tell(); + auto grid = std::unique_ptr<NTv2Grid>( + new NTv2Grid(filename + ", " + gridName, ctx, fpRaw, offset, + must_swap, columns, rows, extent)); std::string parentName; parentName.assign(header + 24, 8); auto iter = mapGrids.find(parentName); @@ -772,11 +1896,293 @@ std::unique_ptr<NTv2GridSet> NTv2GridSet::open(PJ_CONTEXT *ctx, PAFile fp, mapGrids[gridName] = gridPtr; // Skip grid data. 4 components of size float - pj_ctx_fseek(ctx, fp, gs_count * 4 * 4, SEEK_CUR); + fpRaw->seek(static_cast<unsigned long long>(gs_count) * 4 * 4, + SEEK_CUR); } return set; } +#ifdef TIFF_ENABLED + +// --------------------------------------------------------------------------- + +class GTiffHGridShiftSet : public HorizontalShiftGridSet, public GTiffDataset { + + GTiffHGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : GTiffDataset(ctx, std::move(fp)) {} + + public: + ~GTiffHGridShiftSet() override; + + static std::unique_ptr<GTiffHGridShiftSet> + open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + HorizontalShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +class GTiffHGrid : public HorizontalShiftGrid { + friend void insertIntoHierarchy<GTiffHGrid, HorizontalShiftGrid>( + PJ_CONTEXT *ctx, std::unique_ptr<GTiffHGrid> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<HorizontalShiftGrid>> &topGrids, + std::map<std::string, GTiffHGrid *> &mapGrids); + + std::unique_ptr<GTiffGrid> m_grid; + uint16 m_idxLatShift; + uint16 m_idxLonShift; + double m_convFactorToRadian; + bool m_positiveEast; + + public: + GTiffHGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast); + + ~GTiffHGrid() override; + + bool valueAt(int x, int y, bool, float &lonShift, + float &latShift) const override; + + void insertGrid(PJ_CONTEXT *ctx, std::unique_ptr<GTiffHGrid> &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +GTiffHGridShiftSet::~GTiffHGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffHGrid::GTiffHGrid(std::unique_ptr<GTiffGrid> &&grid, uint16 idxLatShift, + uint16 idxLonShift, double convFactorToRadian, + bool positiveEast) + : HorizontalShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), + m_grid(std::move(grid)), m_idxLatShift(idxLatShift), + m_idxLonShift(idxLonShift), m_convFactorToRadian(convFactorToRadian), + m_positiveEast(positiveEast) {} + +// --------------------------------------------------------------------------- + +GTiffHGrid::~GTiffHGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffHGrid::valueAt(int x, int y, bool, float &lonShift, + float &latShift) const { + if (!m_grid->valueAt(m_idxLatShift, x, y, latShift) || + !m_grid->valueAt(m_idxLonShift, x, y, lonShift)) { + return false; + } + // From arc-seconds to radians + latShift = static_cast<float>(latShift * m_convFactorToRadian); + lonShift = static_cast<float>(lonShift * m_convFactorToRadian); + if (!m_positiveEast) { + lonShift = -lonShift; + } + return true; +} + +// --------------------------------------------------------------------------- + +void GTiffHGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffHGrid> &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GTiffHGrid *>(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} + +// --------------------------------------------------------------------------- + +std::unique_ptr<GTiffHGridShiftSet> +GTiffHGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + auto set = std::unique_ptr<GTiffHGridShiftSet>( + new GTiffHGridShiftSet(ctx, std::move(fp))); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + + // Defaults inspired from NTv2 + uint16 idxLatShift = 0; + uint16 idxLonShift = 1; + constexpr double ARC_SECOND_TO_RADIAN = (M_PI / 180.0) / 3600.0; + double convFactorToRadian = ARC_SECOND_TO_RADIAN; + bool positiveEast = true; + + std::map<std::string, GTiffHGrid *> mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + if (grid->samplesPerPixel() < 2) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, + "At least 2 samples per pixel needed"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has not at least 2 samples", ifd); + continue; + } + } + + // Identify the index of the latitude and longitude offset channels + bool foundDescriptionForAtLeastOneSample = false; + bool foundDescriptionForLatOffset = false; + bool foundDescriptionForLonOffset = false; + for (int i = 0; i < static_cast<int>(grid->samplesPerPixel()); ++i) { + const auto desc = grid->metadataItem("DESCRIPTION", i); + if (!desc.empty()) { + foundDescriptionForAtLeastOneSample = true; + } + if (desc == "latitude_offset") { + idxLatShift = static_cast<uint16>(i); + foundDescriptionForLatOffset = true; + } else if (desc == "longitude_offset") { + idxLonShift = static_cast<uint16>(i); + foundDescriptionForLonOffset = true; + } + } + + if (foundDescriptionForAtLeastOneSample) { + if (!foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + if (ifd > 0) { + // Assuming that extra IFD without + // longitude_offset/latitude_offset can be ignored + // One could imagine to put the accuracy values in separate + // IFD for example + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has no " + "longitude_offset/latitude_offset channel", + ifd); + continue; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "IFD 0 has channel descriptions, but no " + "longitude_offset/latitude_offset channel"); + return nullptr; + } + } + } + if (foundDescriptionForLatOffset && !foundDescriptionForLonOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found latitude_offset channel, but not longitude_offset"); + return nullptr; + } else if (foundDescriptionForLonOffset && + !foundDescriptionForLatOffset) { + pj_log(ctx, PJ_LOG_ERROR, + "Found longitude_offset channel, but not latitude_offset"); + return nullptr; + } + + if (idxLatShift >= grid->samplesPerPixel() || + idxLonShift >= grid->samplesPerPixel()) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid sample index"); + return nullptr; + } + + if (foundDescriptionForLonOffset) { + const std::string positiveValue = + grid->metadataItem("positive_value", idxLonShift); + if (!positiveValue.empty()) { + if (positiveValue == "west") { + positiveEast = false; + } else if (positiveValue == "east") { + positiveEast = true; + } else { + pj_log(ctx, PJ_LOG_ERROR, + "Unsupported value %s for 'positive_value'", + positiveValue.c_str()); + return nullptr; + } + } + } + + // Identify their unit + { + const auto unitLatShift = + grid->metadataItem("UNITTYPE", idxLatShift); + const auto unitLonShift = + grid->metadataItem("UNITTYPE", idxLonShift); + if (unitLatShift != unitLonShift) { + pj_log(ctx, PJ_LOG_ERROR, + "Different unit for longitude and latitude offset"); + return nullptr; + } + if (!unitLatShift.empty()) { + if (unitLatShift == "arc-second") { + convFactorToRadian = ARC_SECOND_TO_RADIAN; + } else if (unitLatShift == "radian") { + convFactorToRadian = 1.0; + } else if (unitLatShift == "degree") { + convFactorToRadian = M_PI / 180.0; + } else { + pj_log(ctx, PJ_LOG_ERROR, "Unsupported unit %s", + unitLatShift.c_str()); + return nullptr; + } + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); + + auto hgrid = internal::make_unique<GTiffHGrid>( + std::move(grid), idxLatShift, idxLonShift, convFactorToRadian, + positiveEast); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} +#endif // TIFF_ENABLED + // --------------------------------------------------------------------------- std::unique_ptr<HorizontalShiftGridSet> @@ -791,27 +2197,26 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { return set; } - PAFile fp; - if (!(fp = pj_open_lib(ctx, filename.c_str(), "rb"))) { + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { ctx->last_errno = 0; /* don't treat as a persistent error */ return nullptr; } + const auto actualName(fp->name()); char header[160]; /* -------------------------------------------------------------------- */ /* Load a header, to determine the file type. */ /* -------------------------------------------------------------------- */ - size_t header_size; - if ((header_size = pj_ctx_fread(ctx, header, 1, sizeof(header), fp)) != - sizeof(header)) { + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { /* some files may be smaller that sizeof(header), eg 160, so */ ctx->last_errno = 0; /* don't treat as a persistent error */ pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "pj_gridinfo_init: short header read of %d bytes", (int)header_size); } - - pj_ctx_fseek(ctx, fp, SEEK_SET, 0); + fp->seek(0); /* -------------------------------------------------------------------- */ /* Determine file type. */ @@ -819,37 +2224,46 @@ HorizontalShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { if (header_size >= 144 + 16 && strncmp(header + 0, "HEADER", 6) == 0 && strncmp(header + 96, "W GRID", 6) == 0 && strncmp(header + 144, "TO NAD83 ", 16) == 0) { - auto grid = NTv1Grid::open(ctx, fp, filename); + auto grid = NTv1Grid::open(ctx, std::move(fp), actualName); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = std::unique_ptr<HorizontalShiftGridSet>( new HorizontalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "ntv1"; set->m_grids.push_back(std::unique_ptr<HorizontalShiftGrid>(grid)); return set; } else if (header_size >= 9 && strncmp(header + 0, "CTABLE V2", 9) == 0) { - auto grid = CTable2Grid::open(ctx, fp, filename); + auto grid = CTable2Grid::open(ctx, std::move(fp), actualName); if (!grid) { - pj_ctx_fclose(ctx, fp); return nullptr; } auto set = std::unique_ptr<HorizontalShiftGridSet>( new HorizontalShiftGridSet()); - set->m_name = filename; + set->m_name = actualName; set->m_format = "ctable2"; set->m_grids.push_back(std::unique_ptr<HorizontalShiftGrid>(grid)); return set; } else if (header_size >= 48 + 7 && strncmp(header + 0, "NUM_OREC", 8) == 0 && strncmp(header + 48, "GS_TYPE", 7) == 0) { - return NTv2GridSet::open(ctx, fp, filename); + return NTv2GridSet::open(ctx, std::move(fp), actualName); + } else if (IsTIFF(header_size, + reinterpret_cast<const unsigned char *>(header))) { +#ifdef TIFF_ENABLED + auto set = GTiffHGridShiftSet::open(ctx, std::move(fp), actualName); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; +#else + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); + return nullptr; +#endif } pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized horizontal grid format"); - pj_ctx_fclose(ctx, fp); return nullptr; } @@ -892,4 +2306,342 @@ const HorizontalShiftGrid *HorizontalShiftGridSet::gridAt(double lon, return nullptr; } +// --------------------------------------------------------------------------- + +void HorizontalShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +#ifdef TIFF_ENABLED +// --------------------------------------------------------------------------- + +class GTiffGenericGridShiftSet : public GenericShiftGridSet, + public GTiffDataset { + + GTiffGenericGridShiftSet(PJ_CONTEXT *ctx, std::unique_ptr<File> &&fp) + : GTiffDataset(ctx, std::move(fp)) {} + + public: + ~GTiffGenericGridShiftSet() override; + + static std::unique_ptr<GTiffGenericGridShiftSet> + open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename); + + void reassign_context(PJ_CONTEXT *ctx) override { + GenericShiftGridSet::reassign_context(ctx); + GTiffDataset::reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +class GTiffGenericGrid : public GenericShiftGrid { + friend void insertIntoHierarchy<GTiffGenericGrid, GenericShiftGrid>( + PJ_CONTEXT *ctx, std::unique_ptr<GTiffGenericGrid> &&grid, + const std::string &gridName, const std::string &parentName, + std::vector<std::unique_ptr<GenericShiftGrid>> &topGrids, + std::map<std::string, GTiffGenericGrid *> &mapGrids); + + std::unique_ptr<GTiffGrid> m_grid; + + public: + GTiffGenericGrid(std::unique_ptr<GTiffGrid> &&grid); + + ~GTiffGenericGrid() override; + + bool valueAt(int x, int y, int sample, float &out) const override; + + int samplesPerPixel() const override { return m_grid->samplesPerPixel(); } + + std::string unit(int sample) const override { + return m_grid->metadataItem("UNITTYPE", sample); + } + + std::string description(int sample) const override { + return m_grid->metadataItem("DESCRIPTION", sample); + } + + std::string metadataItem(const std::string &key, + int sample = -1) const override { + return m_grid->metadataItem(key, sample); + } + + void insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffGenericGrid> &&subgrid); + + void reassign_context(PJ_CONTEXT *ctx) override { + m_grid->reassign_context(ctx); + } +}; + +// --------------------------------------------------------------------------- + +GTiffGenericGridShiftSet::~GTiffGenericGridShiftSet() = default; + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::GTiffGenericGrid(std::unique_ptr<GTiffGrid> &&grid) + : GenericShiftGrid(grid->name(), grid->width(), grid->height(), + grid->extentAndRes()), + m_grid(std::move(grid)) {} + +// --------------------------------------------------------------------------- + +GTiffGenericGrid::~GTiffGenericGrid() = default; + +// --------------------------------------------------------------------------- + +bool GTiffGenericGrid::valueAt(int x, int y, int sample, float &out) const { + if (sample < 0 || + static_cast<unsigned>(sample) >= m_grid->samplesPerPixel()) + return false; + return m_grid->valueAt(static_cast<uint16>(sample), x, y, out); +} + +// --------------------------------------------------------------------------- + +void GTiffGenericGrid::insertGrid(PJ_CONTEXT *ctx, + std::unique_ptr<GTiffGenericGrid> &&subgrid) { + bool gridInserted = false; + const auto &extent = subgrid->extentAndRes(); + for (const auto &candidateParent : m_children) { + const auto &candidateParentExtent = candidateParent->extentAndRes(); + if (candidateParentExtent.contains(extent)) { + static_cast<GTiffGenericGrid *>(candidateParent.get()) + ->insertGrid(ctx, std::move(subgrid)); + gridInserted = true; + break; + } else if (candidateParentExtent.intersects(extent)) { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Partially intersecting grids found!"); + } + } + if (!gridInserted) { + m_children.emplace_back(std::move(subgrid)); + } +} +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +class NullGenericShiftGrid : public GenericShiftGrid { + + public: + NullGenericShiftGrid() : GenericShiftGrid("null", 3, 3, globalExtent()) {} + + bool isNullGrid() const override { return true; } + bool valueAt(int, int, int, float &out) const override; + + int samplesPerPixel() const override { return 0; } + + std::string unit(int) const override { return std::string(); } + + std::string description(int) const override { return std::string(); } + + std::string metadataItem(const std::string &, int) const override { + return std::string(); + } + + void reassign_context(PJ_CONTEXT *) override {} +}; + +// --------------------------------------------------------------------------- + +bool NullGenericShiftGrid::valueAt(int, int, int, float &out) const { + out = 0.0f; + return true; +} + +// --------------------------------------------------------------------------- + +#ifdef TIFF_ENABLED + +std::unique_ptr<GTiffGenericGridShiftSet> +GTiffGenericGridShiftSet::open(PJ_CONTEXT *ctx, std::unique_ptr<File> fp, + const std::string &filename) { + auto set = std::unique_ptr<GTiffGenericGridShiftSet>( + new GTiffGenericGridShiftSet(ctx, std::move(fp))); + set->m_name = filename; + set->m_format = "gtiff"; + if (!set->openTIFF(filename)) { + return nullptr; + } + + std::map<std::string, GTiffGenericGrid *> mapGrids; + for (int ifd = 0;; ++ifd) { + auto grid = set->nextGrid(); + if (!grid) { + if (ifd == 0) { + return nullptr; + } + break; + } + + const auto subfileType = grid->subfileType(); + if (subfileType != 0 && subfileType != FILETYPE_PAGE) { + if (ifd == 0) { + pj_log(ctx, PJ_LOG_ERROR, "Invalid subfileType"); + return nullptr; + } else { + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, + "Ignoring IFD %d as it has a unsupported subfileType", + ifd); + continue; + } + } + + const std::string gridName = grid->metadataItem("grid_name"); + const std::string parentName = grid->metadataItem("parent_grid_name"); + + auto hgrid = internal::make_unique<GTiffGenericGrid>(std::move(grid)); + + insertIntoHierarchy(ctx, std::move(hgrid), gridName, parentName, + set->m_grids, mapGrids); + } + return set; +} +#endif // TIFF_ENABLED + +// --------------------------------------------------------------------------- + +GenericShiftGrid::GenericShiftGrid(const std::string &nameIn, int widthIn, + int heightIn, const ExtentAndRes &extentIn) + : Grid(nameIn, widthIn, heightIn, extentIn) {} + +// --------------------------------------------------------------------------- + +GenericShiftGrid::~GenericShiftGrid() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +GenericShiftGridSet::~GenericShiftGridSet() = default; + +// --------------------------------------------------------------------------- + +std::unique_ptr<GenericShiftGridSet> +GenericShiftGridSet::open(PJ_CONTEXT *ctx, const std::string &filename) { + if (filename == "null") { + auto set = + std::unique_ptr<GenericShiftGridSet>(new GenericShiftGridSet()); + set->m_name = filename; + set->m_format = "null"; + set->m_grids.push_back( + std::unique_ptr<NullGenericShiftGrid>(new NullGenericShiftGrid())); + return set; + } + + auto fp = FileManager::open_resource_file(ctx, filename.c_str()); + if (!fp) { + ctx->last_errno = 0; /* don't treat as a persistent error */ + return nullptr; + } + const auto actualName(fp->name()); + + /* -------------------------------------------------------------------- */ + /* Load a header, to determine the file type. */ + /* -------------------------------------------------------------------- */ + unsigned char header[4]; + size_t header_size = fp->read(header, sizeof(header)); + if (header_size != sizeof(header)) { + return nullptr; + } + fp->seek(0); + + if (IsTIFF(header_size, header)) { +#ifdef TIFF_ENABLED + auto set = + GTiffGenericGridShiftSet::open(ctx, std::move(fp), actualName); + if (!set) + pj_ctx_set_errno(ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return set; +#else + pj_log(ctx, PJ_LOG_ERROR, + "TIFF grid, but TIFF support disabled in this build"); + return nullptr; +#endif + } + + pj_log(ctx, PJ_LOG_DEBUG_MAJOR, "Unrecognized generic grid format"); + return nullptr; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGrid::gridAt(double lon, double lat) const { + for (const auto &child : m_children) { + const auto &extentChild = child->extentAndRes(); + if ((extentChild.fullWorldLongitude() || + (lon >= extentChild.westLon && lon <= extentChild.eastLon)) && + lat >= extentChild.southLat && lat <= extentChild.northLat) { + return child->gridAt(lon, lat); + } + } + return this; +} + +// --------------------------------------------------------------------------- + +const GenericShiftGrid *GenericShiftGridSet::gridAt(double lon, + double lat) const { + for (const auto &grid : m_grids) { + if (dynamic_cast<NullGenericShiftGrid *>(grid.get())) { + return grid.get(); + } + const auto &extent = grid->extentAndRes(); + if ((extent.fullWorldLongitude() || + (lon >= extent.westLon && lon <= extent.eastLon)) && + lat >= extent.southLat && lat <= extent.northLat) { + return grid->gridAt(lon, lat); + } + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +void GenericShiftGridSet::reassign_context(PJ_CONTEXT *ctx) { + for (const auto &grid : m_grids) { + grid->reassign_context(ctx); + } +} + +// --------------------------------------------------------------------------- + +ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *gridkey) { + std::string key("s"); + key += gridkey; + const char *gridnames = pj_param(P->ctx, P->params, key.c_str()).s; + if (gridnames == nullptr) + return {}; + + auto listOfGridNames = internal::split(std::string(gridnames), ','); + ListOfGenericGrids grids; + for (const auto &gridnameStr : listOfGridNames) { + const char *gridname = gridnameStr.c_str(); + bool canFail = false; + if (gridname[0] == '@') { + canFail = true; + gridname++; + } + auto gridSet = GenericShiftGridSet::open(P->ctx, gridname); + if (!gridSet) { + if (!canFail) { + pj_ctx_set_errno(P->ctx, PJD_ERR_FAILED_TO_LOAD_GRID); + return {}; + } + } else { + grids.emplace_back(std::move(gridSet)); + } + } + + return grids; +} + NS_PROJ_END diff --git a/src/grids.hpp b/src/grids.hpp index 65bf502b..aa852ef6 100644 --- a/src/grids.hpp +++ b/src/grids.hpp @@ -45,17 +45,21 @@ struct ExtentAndRes { double resLat; // in radian bool fullWorldLongitude() const; + bool contains(const ExtentAndRes &other) const; + bool intersects(const ExtentAndRes &other) const; }; // --------------------------------------------------------------------------- class Grid { protected: + std::string m_name; int m_width; int m_height; ExtentAndRes m_extent; - Grid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + Grid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); public: virtual ~Grid(); @@ -63,6 +67,7 @@ class Grid { int width() const { return m_width; } int height() const { return m_height; } const ExtentAndRes &extentAndRes() const { return m_extent; } + const std::string &name() const { return m_name; } virtual bool isNullGrid() const { return false; } }; @@ -74,7 +79,8 @@ class VerticalShiftGrid : public Grid { std::vector<std::unique_ptr<VerticalShiftGrid>> m_children{}; public: - VerticalShiftGrid(int widthIn, int heightIn, const ExtentAndRes &extentIn); + VerticalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); const VerticalShiftGrid *gridAt(double lon, double lat) const; @@ -82,11 +88,14 @@ class VerticalShiftGrid : public Grid { // x = 0 is western-most column, y = 0 is southern-most line virtual bool valueAt(int x, int y, float &out) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- class VerticalShiftGridSet { + protected: std::string m_name{}; std::string m_format{}; std::vector<std::unique_ptr<VerticalShiftGrid>> m_grids{}; @@ -105,6 +114,8 @@ class VerticalShiftGridSet { return m_grids; } const VerticalShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- @@ -114,15 +125,17 @@ class HorizontalShiftGrid : public Grid { std::vector<std::unique_ptr<HorizontalShiftGrid>> m_children{}; public: - HorizontalShiftGrid(int widthIn, int heightIn, + HorizontalShiftGrid(const std::string &nameIn, int widthIn, int heightIn, const ExtentAndRes &extentIn); ~HorizontalShiftGrid() override; const HorizontalShiftGrid *gridAt(double lon, double lat) const; // x = 0 is western-most column, y = 0 is southern-most line - virtual bool valueAt(int x, int y, float &lonShift, - float &latShift) const = 0; + virtual bool valueAt(int x, int y, bool compensateNTConvention, + float &lonShift, float &latShift) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; }; // --------------------------------------------------------------------------- @@ -147,15 +160,75 @@ class HorizontalShiftGridSet { return m_grids; } const HorizontalShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); +}; + +// --------------------------------------------------------------------------- + +class GenericShiftGrid : public Grid { + protected: + std::vector<std::unique_ptr<GenericShiftGrid>> m_children{}; + + public: + GenericShiftGrid(const std::string &nameIn, int widthIn, int heightIn, + const ExtentAndRes &extentIn); + + ~GenericShiftGrid() override; + + const GenericShiftGrid *gridAt(double lon, double lat) const; + + virtual std::string unit(int sample) const = 0; + + virtual std::string description(int sample) const = 0; + + virtual std::string metadataItem(const std::string &key, + int sample = -1) const = 0; + + virtual int samplesPerPixel() const = 0; + + // x = 0 is western-most column, y = 0 is southern-most line + virtual bool valueAt(int x, int y, int sample, float &out) const = 0; + + virtual void reassign_context(PJ_CONTEXT *ctx) = 0; +}; + +// --------------------------------------------------------------------------- + +class GenericShiftGridSet { + protected: + std::string m_name{}; + std::string m_format{}; + std::vector<std::unique_ptr<GenericShiftGrid>> m_grids{}; + + GenericShiftGridSet(); + + public: + virtual ~GenericShiftGridSet(); + + static std::unique_ptr<GenericShiftGridSet> + open(PJ_CONTEXT *ctx, const std::string &filename); + + const std::string &name() const { return m_name; } + const std::string &format() const { return m_format; } + const std::vector<std::unique_ptr<GenericShiftGrid>> &grids() const { + return m_grids; + } + const GenericShiftGrid *gridAt(double lon, double lat) const; + + virtual void reassign_context(PJ_CONTEXT *ctx); }; // --------------------------------------------------------------------------- typedef std::vector<std::unique_ptr<HorizontalShiftGridSet>> ListOfHGrids; typedef std::vector<std::unique_ptr<VerticalShiftGridSet>> ListOfVGrids; +typedef std::vector<std::unique_ptr<GenericShiftGridSet>> ListOfGenericGrids; ListOfVGrids proj_vgrid_init(PJ *P, const char *grids); ListOfHGrids proj_hgrid_init(PJ *P, const char *grids); +ListOfGenericGrids proj_generic_grid_init(PJ *P, const char *grids); + double proj_vgrid_value(PJ *P, const ListOfVGrids &, PJ_LP lp, double vmultiplier); PJ_LP proj_hgrid_value(PJ *P, const ListOfHGrids &, PJ_LP lp); diff --git a/src/iso19111/c_api.cpp b/src/iso19111/c_api.cpp index 9db9e5b3..5df4c513 100644 --- a/src/iso19111/c_api.cpp +++ b/src/iso19111/c_api.cpp @@ -177,7 +177,11 @@ static PJ *pj_obj_create(PJ_CONTEXT *ctx, const IdentifiedObjectNNPtr &objIn) { auto formatter = PROJStringFormatter::create( PROJStringFormatter::Convention::PROJ_5, dbContext); auto projString = coordop->exportToPROJString(formatter.get()); + if (pj_context_is_network_enabled(ctx)) { + ctx->defer_grid_opening = true; + } auto pj = pj_create_internal(ctx, projString.c_str()); + ctx->defer_grid_opening = false; if (pj) { pj->iso_obj = objIn; if (ctx->cpp_context) { @@ -766,7 +770,7 @@ int PROJ_DLL proj_grid_get_info_from_database( bool open_license; bool available; if (!db_context->lookForGridInfo( - grid_name, ctx->cpp_context->lastGridFullName_, + grid_name, false, ctx->cpp_context->lastGridFullName_, ctx->cpp_context->lastGridPackageName_, ctx->cpp_context->lastGridUrl_, direct_download, open_license, available)) { @@ -6571,7 +6575,10 @@ int proj_coordoperation_is_instantiable(PJ_CONTEXT *ctx, } auto dbContext = getDBcontextNoException(ctx, __FUNCTION__); try { - auto ret = op->isPROJInstantiable(dbContext) ? 1 : 0; + auto ret = op->isPROJInstantiable(dbContext, + pj_context_is_network_enabled(ctx)) + ? 1 + : 0; if (ctx->cpp_context) { ctx->cpp_context->autoCloseDbIfNeeded(); } @@ -6883,7 +6890,8 @@ int proj_coordoperation_get_grid_used_count(PJ_CONTEXT *ctx, try { if (!coordoperation->gridsNeededAsked) { coordoperation->gridsNeededAsked = true; - const auto gridsNeeded = co->gridsNeeded(dbContext); + const auto gridsNeeded = + co->gridsNeeded(dbContext, pj_context_is_network_enabled(ctx)); for (const auto &gridDesc : gridsNeeded) { coordoperation->gridsNeeded.emplace_back(gridDesc); } @@ -7220,6 +7228,12 @@ void PROJ_DLL proj_operation_factory_context_set_grid_availability_use( CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY); break; + + case PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE: + factory_ctx->operationContext->setGridAvailabilityUse( + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); + break; } } catch (const std::exception &e) { proj_log_error(ctx, __FUNCTION__, e.what()); diff --git a/src/iso19111/coordinateoperation.cpp b/src/iso19111/coordinateoperation.cpp index 6120c768..7c0515c7 100644 --- a/src/iso19111/coordinateoperation.cpp +++ b/src/iso19111/coordinateoperation.cpp @@ -788,13 +788,15 @@ void CoordinateOperation::setAccuracies( * available. */ bool CoordinateOperation::isPROJInstantiable( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { try { exportToPROJString(io::PROJStringFormatter::create().get()); } catch (const std::exception &) { return false; } - for (const auto &gridDesc : gridsNeeded(databaseContext)) { + for (const auto &gridDesc : + gridsNeeded(databaseContext, considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return false; } @@ -2013,8 +2015,9 @@ bool SingleOperation::_isEquivalentTo(const util::IComparable *other, // --------------------------------------------------------------------------- -std::set<GridDescription> SingleOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set<GridDescription> +SingleOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set<GridDescription> res; for (const auto &genOpParamvalue : parameterValues()) { auto opParamvalue = dynamic_cast<const OperationParameterValue *>( @@ -2026,9 +2029,9 @@ std::set<GridDescription> SingleOperation::gridsNeeded( desc.shortName = value->valueFile(); if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, - desc.url, desc.directDownload, desc.openLicense, - desc.available); + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, + desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); } @@ -10209,10 +10212,12 @@ bool ConcatenatedOperation::_isEquivalentTo( // --------------------------------------------------------------------------- std::set<GridDescription> ConcatenatedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { + const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set<GridDescription> res; for (const auto &operation : operations()) { - const auto l_gridsNeeded = operation->gridsNeeded(databaseContext); + const auto l_gridsNeeded = operation->gridsNeeded( + databaseContext, considerKnownGridsAsAvailable); for (const auto &gridDesc : l_gridsNeeded) { res.insert(gridDesc); } @@ -11132,7 +11137,10 @@ struct FilterResults { bool gridsKnown = true; if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { hasGrids = true; if (gridAvailabilityUse == @@ -11254,6 +11262,7 @@ struct FilterResults { CoordinateOperationPtr lastOp; bool first = true; + const auto gridAvailabilityUse = context->getGridAvailabilityUse(); for (const auto &op : res) { const auto curAccuracy = getAccuracy(op); bool dummy = false; @@ -11266,7 +11275,10 @@ struct FilterResults { if (context->getAuthorityFactory()) { const auto gridsNeeded = op->gridsNeeded( - context->getAuthorityFactory()->databaseContext()); + context->getAuthorityFactory()->databaseContext(), + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { curHasGrids = true; curSetOfGrids.insert(gridDesc.shortName); @@ -11571,6 +11583,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11590,9 +11603,16 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirect( tmpAuthFactory->createFromCoordinateReferenceSystemCodes( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, false, context.extent1, context.extent2); res.insert(res.end(), resTmp.begin(), resTmp.end()); @@ -11635,6 +11655,7 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( std::list<std::pair<std::string, std::string>> ids; buildCRSIds(targetCRS, context, ids); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &id : ids) { const auto &targetAuthName = id.first; const auto &targetCode = id.second; @@ -11648,9 +11669,15 @@ CoordinateOperationFactory::Private::findOpsInRegistryDirectTo( auto res = tmpAuthFactory->createFromCoordinateReferenceSystemCodes( std::string(), std::string(), targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), true, true, context.extent1, context.extent2); if (!res.empty()) { @@ -11698,6 +11725,7 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( buildCRSIds(sourceCRS, context, sourceIds); buildCRSIds(targetCRS, context, targetIds); + const auto gridAvailabilityUse = context.context->getGridAvailabilityUse(); for (const auto &idSrc : sourceIds) { const auto &srcAuthName = idSrc.first; const auto &srcCode = idSrc.second; @@ -11717,21 +11745,28 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( std::vector<CoordinateOperationNNPtr> res; if (useCreateBetweenGeodeticCRSWithDatumBasedIntermediates) { - res = tmpAuthFactory - ->createBetweenGeodeticCRSWithDatumBasedIntermediates( - sourceCRS, srcAuthName, srcCode, targetCRS, - targetAuthName, targetCode, - context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == - CoordinateOperationContext:: - GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, - context.context->getDiscardSuperseded(), - authFactory->getAuthority() != "any" && - authorities.size() > 1 - ? authorities - : std::vector<std::string>(), - context.extent1, context.extent2); + res = + tmpAuthFactory + ->createBetweenGeodeticCRSWithDatumBasedIntermediates( + sourceCRS, srcAuthName, srcCode, targetCRS, + targetAuthName, targetCode, + context.context->getUsePROJAlternativeGridNames(), + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE, + context.context->getDiscardSuperseded(), + authFactory->getAuthority() != "any" && + authorities.size() > 1 + ? authorities + : std::vector<std::string>(), + context.extent1, context.extent2); } else { io::AuthorityFactory::ObjectType intermediateObjectType = io::AuthorityFactory::ObjectType::CRS; @@ -11749,9 +11784,15 @@ CoordinateOperationFactory::Private::findsOpsInRegistryWithIntermediate( res = tmpAuthFactory->createFromCRSCodesWithIntermediates( srcAuthName, srcCode, targetAuthName, targetCode, context.context->getUsePROJAlternativeGridNames(), - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + DISCARD_OPERATION_IF_MISSING_GRID || + gridAvailabilityUse == + CoordinateOperationContext::GridAvailabilityUse:: + KNOWN_AVAILABLE, + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: - DISCARD_OPERATION_IF_MISSING_GRID, + KNOWN_AVAILABLE, context.context->getDiscardSuperseded(), context.context->getIntermediateCRS(), intermediateObjectType, @@ -14167,15 +14208,21 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( componentsSrc[1], targetCRS->promoteTo3D(std::string(), dbContext), context); bool foundRegisteredTransformWithAllGridsAvailable = false; + const auto gridAvailabilityUse = + context.context->getGridAvailabilityUse(); const bool ignoreMissingGrids = - context.context->getGridAvailabilityUse() == + gridAvailabilityUse == CoordinateOperationContext::GridAvailabilityUse:: IGNORE_GRID_AVAILABILITY; for (const auto &op : verticalTransforms) { if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -14204,7 +14251,11 @@ void CoordinateOperationFactory::Private::createOperationsCompoundToGeog( if (hasIdentifiers(op) && dbContext) { bool missingGrid = false; if (!ignoreMissingGrids) { - const auto gridsNeeded = op->gridsNeeded(dbContext); + const auto gridsNeeded = op->gridsNeeded( + dbContext, + gridAvailabilityUse == + CoordinateOperationContext:: + GridAvailabilityUse::KNOWN_AVAILABLE); for (const auto &gridDesc : gridsNeeded) { if (!gridDesc.available) { missingGrid = true; @@ -15034,8 +15085,9 @@ CoordinateOperationNNPtr PROJBasedOperation::_shallowClone() const { // --------------------------------------------------------------------------- -std::set<GridDescription> PROJBasedOperation::gridsNeeded( - const io::DatabaseContextPtr &databaseContext) const { +std::set<GridDescription> +PROJBasedOperation::gridsNeeded(const io::DatabaseContextPtr &databaseContext, + bool considerKnownGridsAsAvailable) const { std::set<GridDescription> res; try { @@ -15048,7 +15100,8 @@ std::set<GridDescription> PROJBasedOperation::gridsNeeded( desc.shortName = shortName; if (databaseContext) { databaseContext->lookForGridInfo( - desc.shortName, desc.fullName, desc.packageName, desc.url, + desc.shortName, considerKnownGridsAsAvailable, + desc.fullName, desc.packageName, desc.url, desc.directDownload, desc.openLicense, desc.available); } res.insert(desc); diff --git a/src/iso19111/factory.cpp b/src/iso19111/factory.cpp index 57850303..dae8680c 100644 --- a/src/iso19111/factory.cpp +++ b/src/iso19111/factory.cpp @@ -1018,14 +1018,14 @@ bool DatabaseContext::lookForGridAlternative(const std::string &officialName, // --------------------------------------------------------------------------- -bool DatabaseContext::lookForGridInfo(const std::string &projFilename, - std::string &fullFilename, - std::string &packageName, - std::string &url, bool &directDownload, - bool &openLicense, - bool &gridAvailable) const { +bool DatabaseContext::lookForGridInfo( + const std::string &projFilename, bool considerKnownGridsAsAvailable, + std::string &fullFilename, std::string &packageName, std::string &url, + bool &directDownload, bool &openLicense, bool &gridAvailable) const { Private::GridInfoCache info; - if (d->getGridInfoFromCache(projFilename, info)) { + const std::string key(projFilename + + (considerKnownGridsAsAvailable ? "true" : "false")); + if (d->getGridInfoFromCache(key, info)) { fullFilename = info.fullFilename; packageName = info.packageName; url = info.url; @@ -1041,16 +1041,20 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = false; directDownload = false; - fullFilename.resize(2048); - if (d->pjCtxt() == nullptr) { - d->setPjCtxt(pj_get_default_ctx()); + if (considerKnownGridsAsAvailable) { + fullFilename = projFilename; + } else { + fullFilename.resize(2048); + if (d->pjCtxt() == nullptr) { + d->setPjCtxt(pj_get_default_ctx()); + } + int errno_before = proj_context_errno(d->pjCtxt()); + gridAvailable = + pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], + fullFilename.size() - 1) != 0; + proj_context_errno_set(d->pjCtxt(), errno_before); + fullFilename.resize(strlen(fullFilename.c_str())); } - int errno_before = proj_context_errno(d->pjCtxt()); - gridAvailable = - pj_find_file(d->pjCtxt(), projFilename.c_str(), &fullFilename[0], - fullFilename.size() - 1) != 0; - proj_context_errno_set(d->pjCtxt(), errno_before); - fullFilename.resize(strlen(fullFilename.c_str())); auto res = d->run("SELECT " @@ -1074,6 +1078,10 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, openLicense = (row[3].empty() ? row[4] : row[3]) == "1"; directDownload = (row[5].empty() ? row[6] : row[5]) == "1"; + if (considerKnownGridsAsAvailable && !packageName.empty()) { + gridAvailable = true; + } + info.fullFilename = fullFilename; info.packageName = packageName; info.url = url; @@ -1082,7 +1090,7 @@ bool DatabaseContext::lookForGridInfo(const std::string &projFilename, } info.gridAvailable = gridAvailable; info.found = ret; - d->cache(projFilename, info); + d->cache(key, info); return ret; } @@ -1264,8 +1272,8 @@ struct AuthorityFactory::Private { return AuthorityFactory::create(context_, auth_name); } - bool - rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op); + bool rejectOpDueToMissingGrid(const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable); UnitOfMeasure createUnitOfMeasure(const std::string &auth_name, const std::string &code); @@ -1392,8 +1400,10 @@ util::PropertyMap AuthorityFactory::Private::createProperties( // --------------------------------------------------------------------------- bool AuthorityFactory::Private::rejectOpDueToMissingGrid( - const operation::CoordinateOperationNNPtr &op) { - for (const auto &gridDesc : op->gridsNeeded(context())) { + const operation::CoordinateOperationNNPtr &op, + bool considerKnownGridsAsAvailable) { + for (const auto &gridDesc : + op->gridsNeeded(context(), considerKnownGridsAsAvailable)) { if (!gridDesc.available) { return true; } @@ -3381,7 +3391,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSCode, const std::string &targetCRSCode) const { return createFromCoordinateReferenceSystemCodes( d->authority(), sourceCRSCode, d->authority(), targetCRSCode, false, - false, false); + false, false, false); } // --------------------------------------------------------------------------- @@ -3410,6 +3420,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param tryReverseOrder whether to search in the reverse order too (and thus @@ -3430,8 +3442,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, bool tryReverseOrder, - bool reportOnlyIntersectingTransformations, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + bool tryReverseOrder, bool reportOnlyIntersectingTransformations, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -3442,6 +3454,7 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( cacheKey += targetCRSCode; cacheKey += (usePROJAlternativeGridNames ? '1' : '0'); cacheKey += (discardIfMissingGrid ? '1' : '0'); + cacheKey += (considerKnownGridsAsAvailable ? '1' : '0'); cacheKey += (discardSuperseded ? '1' : '0'); cacheKey += (tryReverseOrder ? '1' : '0'); cacheKey += (reportOnlyIntersectingTransformations ? '1' : '0'); @@ -3680,7 +3693,8 @@ AuthorityFactory::createFromCoordinateReferenceSystemCodes( target_crs_code != targetCRSCode))) { op = op->inverse(); } - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -3745,6 +3759,8 @@ static bool useIrrelevantPivot(const operation::CoordinateOperationNNPtr &op, * should be substituted to the official grid names. * @param discardIfMissingGrid Whether coordinate operations that reference * missing grids should be removed from the result set. + * @param considerKnownGridsAsAvailable Whether known grids should be considered + * as available (typically when network is enabled). * @param discardSuperseded Whether cordinate operations that are superseded * (but not deprecated) should be removed from the result set. * @param intermediateCRSAuthCodes List of (auth_name, code) of CRS that can be @@ -3773,7 +3789,7 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( const std::string &sourceCRSAuthName, const std::string &sourceCRSCode, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, + bool considerKnownGridsAsAvailable, bool discardSuperseded, const std::vector<std::pair<std::string, std::string>> &intermediateCRSAuthCodes, ObjectType allowedIntermediateObjectType, @@ -4221,7 +4237,8 @@ AuthorityFactory::createFromCRSCodesWithIntermediates( std::vector<operation::CoordinateOperationNNPtr> list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } @@ -4239,7 +4256,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( const std::string &sourceCRSCode, const crs::CRSNNPtr &targetCRS, const std::string &targetCRSAuthName, const std::string &targetCRSCode, bool usePROJAlternativeGridNames, bool discardIfMissingGrid, - bool discardSuperseded, const std::vector<std::string> &allowedAuthorities, + bool considerKnownGridsAsAvailable, bool discardSuperseded, + const std::vector<std::string> &allowedAuthorities, const metadata::ExtentPtr &intersectingExtent1, const metadata::ExtentPtr &intersectingExtent2) const { @@ -4822,7 +4840,8 @@ AuthorityFactory::createBetweenGeodeticCRSWithDatumBasedIntermediates( std::vector<operation::CoordinateOperationNNPtr> list; for (const auto &op : listTmp) { - if (!discardIfMissingGrid || !d->rejectOpDueToMissingGrid(op)) { + if (!discardIfMissingGrid || + !d->rejectOpDueToMissingGrid(op, considerKnownGridsAsAvailable)) { list.emplace_back(op); } } diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index f6112aef..12dcb366 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -198,6 +198,7 @@ set(SRC_LIBPROJ_TRANSFORMATIONS transformations/horner.cpp transformations/molodensky.cpp transformations/vgridshift.cpp + transformations/xyzgridshift.cpp ) set(SRC_LIBPROJ_ISO19111 @@ -281,6 +282,8 @@ set(SRC_LIBPROJ_CORE tracing.cpp grids.hpp grids.cpp + filemanager.hpp + filemanager.cpp ${CMAKE_CURRENT_BINARY_DIR}/proj_config.h ) @@ -438,6 +441,16 @@ endif() include_directories(${SQLITE3_INCLUDE_DIR}) target_link_libraries(${PROJ_CORE_TARGET} ${SQLITE3_LIBRARY}) +if(NOT DISABLE_TIFF_IS_STRONGLY_DISCOURAGED) + include_directories(${TIFF_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${TIFF_LIBRARY}) +endif() + +if(CURL_FOUND) + include_directories(${CURL_INCLUDE_DIR}) + target_link_libraries(${PROJ_CORE_TARGET} ${CURL_LIBRARY}) +endif() + if(MSVC) target_compile_definitions(${PROJ_CORE_TARGET} PRIVATE PROJ_MSVC_DLL_EXPORT=1) diff --git a/src/malloc.cpp b/src/malloc.cpp index 3fd3699f..1c539b6b 100644 --- a/src/malloc.cpp +++ b/src/malloc.cpp @@ -49,6 +49,7 @@ #include "proj.h" #include "proj_internal.h" #include "grids.hpp" +#include "filemanager.hpp" using namespace NS_PROJ; @@ -262,4 +263,5 @@ void proj_cleanup() { /*****************************************************************************/ pj_clear_initcache(); pj_deallocate_grids(); + FileManager::clearCache(); } diff --git a/src/open_lib.cpp b/src/open_lib.cpp index 24c31033..cde5be7b 100644 --- a/src/open_lib.cpp +++ b/src/open_lib.cpp @@ -44,6 +44,7 @@ #include "proj/internal/internal.hpp" #include "proj_internal.h" +#include "filemanager.hpp" static const char * proj_lib_name = #ifdef PROJ_LIB @@ -52,6 +53,8 @@ PROJ_LIB; nullptr; #endif +using namespace NS_PROJ::internal; + /************************************************************************/ /* pj_set_finder() */ /************************************************************************/ @@ -197,24 +200,39 @@ static const char *get_path_from_win32_projlib(const char *name, std::string& ou #endif /************************************************************************/ -/* pj_open_lib_ex() */ +/* pj_open_lib_internal() */ /************************************************************************/ -static PAFile -pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, - char* out_full_filename, size_t out_full_filename_size) { - try { - std::string fname; - const char *sysname = nullptr; - PAFile fid = nullptr; #ifdef WIN32 - static const char dir_chars[] = "/\\"; - const char dirSeparator = ';'; +static const char dir_chars[] = "/\\"; +static const char dirSeparator = ';'; #else - static const char dir_chars[] = "/"; - const char dirSeparator = ':'; +static const char dir_chars[] = "/"; +static const char dirSeparator = ':'; #endif +static bool is_tilde_slash(const char* name) +{ + return *name == '~' && strchr(dir_chars,name[1]); +} + +static bool is_rel_or_absolute_filename(const char *name) +{ + return strchr(dir_chars,*name) + || (*name == '.' && strchr(dir_chars,name[1])) + || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) + || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])); +} + +static void* +pj_open_lib_internal(projCtx ctx, const char *name, const char *mode, + void* (*open_file)(projCtx, const char*, const char*), + char* out_full_filename, size_t out_full_filename_size) { + try { + std::string fname; + const char *sysname = nullptr; + void* fid = nullptr; + if( ctx == nullptr ) { ctx = pj_get_default_ctx(); } @@ -223,7 +241,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, out_full_filename[0] = '\0'; /* check if ~/name */ - if (*name == '~' && strchr(dir_chars,name[1]) ) + if (is_tilde_slash(name)) if ((sysname = getenv("HOME")) != nullptr) { fname = sysname; fname += DIR_CHAR; @@ -232,11 +250,10 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, } else return nullptr; - /* or fixed path: /name, ./name or ../name */ - else if (strchr(dir_chars,*name) - || (*name == '.' && strchr(dir_chars,name[1])) - || (!strncmp(name, "..", 2) && strchr(dir_chars,name[2])) - || (name[0] != '\0' && name[1] == ':' && strchr(dir_chars,name[2])) ) + /* or fixed path: /name, ./name or ../name or http[s]:// */ + else if (is_rel_or_absolute_filename(name) + || starts_with(name, "http://") + || starts_with(name, "https://")) sysname = name; /* or try to use application provided file finder */ @@ -254,7 +271,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, fname += DIR_CHAR; fname += name; sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); + fid = open_file(ctx, sysname, mode); } catch( const std::exception& ) { } @@ -270,7 +287,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, fname += DIR_CHAR; fname += name; sysname = fname.c_str(); - fid = pj_ctx_fopen(ctx, sysname, mode); + fid = open_file(ctx, sysname, mode); if( fid ) break; } @@ -290,7 +307,7 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, } assert(sysname); // to make Coverity Scan happy - if ( fid != nullptr || (fid = pj_ctx_fopen(ctx, sysname, mode)) != nullptr) + if ( fid != nullptr || (fid = open_file(ctx, sysname, mode)) != nullptr) { if( out_full_filename != nullptr && out_full_filename_size > 0 ) { @@ -322,18 +339,79 @@ pj_open_lib_ex(projCtx ctx, const char *name, const char *mode, } /************************************************************************/ +/* pj_open_file_with_manager() */ +/************************************************************************/ + +static void* pj_open_file_with_manager(projCtx ctx, const char *name, + const char * /* mode */) +{ + return NS_PROJ::FileManager::open(ctx, name).release(); +} + +/************************************************************************/ +/* FileManager::open_resource_file() */ +/************************************************************************/ + +std::unique_ptr<NS_PROJ::File> NS_PROJ::FileManager::open_resource_file( + projCtx ctx, const char *name) +{ + auto file = std::unique_ptr<NS_PROJ::File>( + reinterpret_cast<NS_PROJ::File*>( + pj_open_lib_internal(ctx, name, "rb", + pj_open_file_with_manager, + nullptr, 0))); + if( file == nullptr && + !is_tilde_slash(name) && + !is_rel_or_absolute_filename(name) && + !starts_with(name, "http://") && + !starts_with(name, "https://") && + pj_context_is_network_enabled(ctx) ) { + std::string remote_file(pj_context_get_url_endpoint(ctx)); + if( !remote_file.empty() ) { + if( remote_file.back() != '/' ) { + remote_file += '/'; + } + remote_file += name; + auto pos = remote_file.rfind('.'); + if( pos + 4 == remote_file.size() ) { + remote_file = remote_file.substr(0, pos) + ".tif"; + file = open(ctx, remote_file.c_str()); + if( file ) { + pj_log( ctx, PJ_LOG_DEBUG_MAJOR, + "Using %s", remote_file.c_str() ); + pj_ctx_set_errno( ctx, 0 ); + } + } + } + } + return file; +} + +/************************************************************************/ /* pj_open_lib() */ /************************************************************************/ +#ifndef REMOVE_LEGACY_SUPPORT + +// Used by following legacy function +static void* pj_ctx_fopen_adapter(projCtx ctx, const char *name, const char *mode) +{ + return pj_ctx_fopen(ctx, name, mode); +} + +// Legacy function PAFile pj_open_lib(projCtx ctx, const char *name, const char *mode) { - return pj_open_lib_ex(ctx, name, mode, nullptr, 0); + return (PAFile)pj_open_lib_internal(ctx, name, mode, pj_ctx_fopen_adapter, nullptr, 0); } +#endif // REMOVE_LEGACY_SUPPORT + /************************************************************************/ /* pj_find_file() */ /************************************************************************/ + /** Returns the full filename corresponding to a proj resource file specified * as a short filename. * @@ -348,12 +426,103 @@ pj_open_lib(projCtx ctx, const char *name, const char *mode) { int pj_find_file(projCtx ctx, const char *short_filename, char* out_full_filename, size_t out_full_filename_size) { - PAFile f = pj_open_lib_ex(ctx, short_filename, "rb", out_full_filename, - out_full_filename_size); + auto f = reinterpret_cast<NS_PROJ::File*>( + pj_open_lib_internal(ctx, short_filename, "rb", + pj_open_file_with_manager, + out_full_filename, + out_full_filename_size)); if( f != nullptr ) { - pj_ctx_fclose(ctx, f); + delete f; return 1; } return 0; } + +/************************************************************************/ +/* pj_context_get_url_endpoint() */ +/************************************************************************/ + +std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx) +{ + if( !ctx->endpoint.empty() ) { + return ctx->endpoint; + } + pj_load_ini(ctx); + return ctx->endpoint; +} + +/************************************************************************/ +/* trim() */ +/************************************************************************/ + +static std::string trim(const std::string& s) { + const auto first = s.find_first_not_of(' '); + const auto last = s.find_last_not_of(' '); + if( first == std::string::npos || last == std::string::npos ) { + return std::string(); + } + return s.substr(first, last - first + 1); +} + +/************************************************************************/ +/* pj_load_ini() */ +/************************************************************************/ + +void pj_load_ini(projCtx ctx) +{ + if( ctx->iniFileLoaded ) + return; + + const char* endpoint_from_env = getenv("PROJ_NETWORK_ENDPOINT"); + if( endpoint_from_env && endpoint_from_env[0] != '\0' ) { + ctx->endpoint = endpoint_from_env; + } + + ctx->iniFileLoaded = true; + auto file = std::unique_ptr<NS_PROJ::File>( + reinterpret_cast<NS_PROJ::File*>( + pj_open_lib_internal(ctx, "proj.ini", "rb", + pj_open_file_with_manager, + nullptr, 0))); + if( !file ) + return; + file->seek(0, SEEK_END); + const auto filesize = file->tell(); + if( filesize == 0 || filesize > 100 * 1024U ) + return; + file->seek(0, SEEK_SET); + std::string content; + content.resize(static_cast<size_t>(filesize)); + const auto nread = file->read(&content[0], content.size()); + if( nread != content.size() ) + return; + content += '\n'; + size_t pos = 0; + while( pos != std::string::npos ) { + const auto eol = content.find_first_of("\r\n", pos); + if( eol == std::string::npos ) { + break; + } + + const auto equal = content.find('=', pos); + if( equal < eol ) + { + const auto key = trim(content.substr(pos, equal-pos)); + const auto value = trim(content.substr(equal + 1, + eol - (equal+1))); + if( ctx->endpoint.empty() && key == "cdn_endpoint" ) { + ctx->endpoint = value; + } else if ( key == "network" ) { + const char *enabled = getenv("PROJ_NETWORK"); + if (enabled == nullptr || enabled[0] == '\0') { + ctx->networking.enabled = ci_equal(value, "ON") || + ci_equal(value, "YES") || + ci_equal(value, "TRUE"); + } + } + } + + pos = content.find_first_not_of("\r\n", eol); + } +} diff --git a/src/pipeline.cpp b/src/pipeline.cpp index 96767143..f65dbfa0 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -155,7 +155,7 @@ static PJ_LPZ pipeline_reverse_3d (PJ_XYZ xyz, PJ *P); static PJ_XY pipeline_forward (PJ_LP lp, PJ *P); static PJ_LP pipeline_reverse (PJ_XY xy, PJ *P); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ) +static void pipeline_reassign_context( PJ* P, PJ_CONTEXT* ctx ) { auto pipeline = static_cast<struct Pipeline*>(P->opaque); for( auto& step: pipeline->steps ) @@ -413,7 +413,7 @@ PJ *OPERATION(pipeline,0) { P->fwd = pipeline_forward; P->inv = pipeline_reverse; P->destructor = destructor; - P->is_pipeline = 1; + P->reassign_context = pipeline_reassign_context; /* Currently, the pipeline driver is a raw bit mover, enabling other operations */ /* to collaborate efficiently. All prep/fin stuff is done at the step levels. */ diff --git a/src/pj_list.h b/src/pj_list.h index 0923bba8..9798a36b 100644 --- a/src/pj_list.h +++ b/src/pj_list.h @@ -170,3 +170,4 @@ PROJ_HEAD(weren, "Werenskiold I") PROJ_HEAD(wink1, "Winkel I") PROJ_HEAD(wink2, "Winkel II") PROJ_HEAD(wintri, "Winkel Tripel") +PROJ_HEAD(xyzgridshift, "XYZ grid shift") @@ -353,6 +353,85 @@ void PROJ_DLL proj_context_set_search_paths(PJ_CONTEXT *ctx, int count_paths, co void PROJ_DLL proj_context_use_proj4_init_rules(PJ_CONTEXT *ctx, int enable); int PROJ_DLL proj_context_get_use_proj4_init_rules(PJ_CONTEXT *ctx, int from_legacy_code_path); +/*! @endcond */ + +/** Opaque structure for PROJ. Implementations might cast it to their + * structure/class of choice. */ +typedef struct PROJ_NETWORK_HANDLE PROJ_NETWORK_HANDLE; + +/** Network access: open callback + * + * Should try to read the size_to_read first bytes at the specified offset of + * the file given by URL url, + * and write them to buffer. *out_size_read should be updated with the actual + * amount of bytes read (== size_to_read if the file is larger than size_to_read). + * During this read, the implementation should make sure to store the HTTP + * headers from the server response to be able to respond to + * proj_network_get_header_value_cbk_type callback. + * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * + * @return a non-NULL opaque handle in case of success. + */ +typedef PROJ_NETWORK_HANDLE* (*proj_network_open_cbk_type)( + PJ_CONTEXT* ctx, + const char* url, + unsigned long long offset, + size_t size_to_read, + void* buffer, + size_t* out_size_read, + size_t error_string_max_size, + char* out_error_string, + void* user_data); + +/** Network access: close callback */ +typedef void (*proj_network_close_cbk_type)(PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + void* user_data); + +/** Network access: get HTTP headers */ +typedef const char* (*proj_network_get_header_value_cbk_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + const char* header_name, + void* user_data); + +/** Network access: read range + * + * Read size_to_read bytes from handle, starting at offset, into + * buffer. + * + * error_string_max_size should be the maximum size that can be written into + * the out_error_string buffer (including terminating nul character). + * + * @return the number of bytes actually read (0 in case of error) + */ +typedef size_t (*proj_network_read_range_type)( + PJ_CONTEXT* ctx, + PROJ_NETWORK_HANDLE* handle, + unsigned long long offset, + size_t size_to_read, + void* buffer, + size_t error_string_max_size, + char* out_error_string, + void* user_data); + +int PROJ_DLL proj_context_set_network_callbacks( + PJ_CONTEXT* ctx, + proj_network_open_cbk_type open_cbk, + proj_network_close_cbk_type close_cbk, + proj_network_get_header_value_cbk_type get_header_value_cbk, + proj_network_read_range_type read_range_cbk, + void* user_data); + +int PROJ_DLL proj_context_set_enable_network(PJ_CONTEXT* ctx, + int enabled); + +void PROJ_DLL proj_context_set_url_endpoint(PJ_CONTEXT* ctx, const char* url); + +/*! @cond Doxygen_Suppress */ + /* Manage the transformation definition object PJ */ PJ PROJ_DLL *proj_create (PJ_CONTEXT *ctx, const char *definition); PJ PROJ_DLL *proj_create_argv (PJ_CONTEXT *ctx, int argc, char **argv); @@ -624,6 +703,12 @@ typedef enum { /** Ignore grid availability at all. Results will be presented as if * all grids were available. */ PROJ_GRID_AVAILABILITY_IGNORED, + + /** Results will be presented as if grids known to PROJ (that is + * registered in the grid_alternatives table of its database) were + * available. Used typically when networking is enabled. + */ + PROJ_GRID_AVAILABILITY_KNOWN_AVAILABLE, } PROJ_GRID_AVAILABILITY_USE; /** \brief PROJ string version. */ diff --git a/src/proj_internal.h b/src/proj_internal.h index 7d826414..12ada034 100644 --- a/src/proj_internal.h +++ b/src/proj_internal.h @@ -214,9 +214,6 @@ size_t pj_trim_argc (char *args); char **pj_trim_argv (size_t argc, char *args); char *pj_make_args (size_t argc, char **argv); -/* Lowest level: Minimum support for fileapi */ -void proj_fileapi_set (PJ *P, void *fileapi); - typedef struct { double r, i; } COMPLEX; /* Forward declarations and typedefs for stuff needed inside the PJ object */ @@ -346,6 +343,7 @@ struct PJconsts { PJ_OPERATOR inv4d = nullptr; PJ_DESTRUCTOR destructor = nullptr; + void (*reassign_context)(PJ*, projCtx_t *) = nullptr; /************************************************************************************* @@ -413,7 +411,6 @@ struct PJconsts { int geoc = 0; /* Geocentric latitude flag */ int is_latlong = 0; /* proj=latlong ... not really a projection at all */ int is_geocent = 0; /* proj=geocent ... not really a projection at all */ - int is_pipeline = 0; /* 1 if PJ represents a pipeline */ int need_ellps = 0; /* 0 for operations that are purely cartesian */ int skip_fwd_prepare = 0; int skip_fwd_finalize = 0; @@ -664,17 +661,29 @@ struct FACTORS { /* NOTE: Remember to update src/strerrno.cpp, src/apps/gie.cpp and transient_error in */ /* src/transform.cpp when adding new value */ +// Legacy struct projFileAPI_t; struct projCppContext; +struct projNetworkCallbacksAndData +{ + bool enabled = false; + bool enabled_env_variable_checked = false; // whereas we have checked PROJ_NETWORK env variable + proj_network_open_cbk_type open = nullptr; + proj_network_close_cbk_type close = nullptr; + proj_network_get_header_value_cbk_type get_header_value = nullptr; + proj_network_read_range_type read_range = nullptr; + void* user_data = nullptr; +}; + /* proj thread context */ struct projCtx_t { int last_errno = 0; int debug_level = 0; void (*logger)(void *, int, const char *) = nullptr; void *logger_app_data = nullptr; - struct projFileAPI_t *fileapi = nullptr; + struct projFileAPI_t *fileapi_legacy = nullptr; // for proj_api.h legacy API struct projCppContext* cpp_context = nullptr; /* internal context for C++ code */ int use_proj4_init_rules = -1; /* -1 = unknown, 0 = no, 1 = yes */ int epsg_file_exists = -1; /* -1 = unknown, 0 = no, 1 = yes */ @@ -686,6 +695,12 @@ struct projCtx_t { const char* (*file_finder) (PJ_CONTEXT *, const char*, void* user_data) = nullptr; void* file_finder_user_data = nullptr; + projNetworkCallbacksAndData networking{}; + bool defer_grid_opening = false; // set by pj_obj_create() + + bool iniFileLoaded = false; + std::string endpoint{}; + int projStringParserCreateFromPROJStringRecursionCounter = 0; // to avoid potential infinite recursion in PROJStringParser::createFromPROJString() projCtx_t() = default; @@ -812,7 +827,12 @@ std::string pj_double_quote_string_param_if_needed(const std::string& str); PJ *pj_create_internal (PJ_CONTEXT *ctx, const char *definition); PJ *pj_create_argv_internal (PJ_CONTEXT *ctx, int argc, char **argv); -void pj_pipeline_assign_context_to_steps( PJ* P, PJ_CONTEXT* ctx ); +// For use by projinfo +bool PROJ_DLL pj_context_is_network_enabled(PJ_CONTEXT* ctx); + +std::string pj_context_get_url_endpoint(PJ_CONTEXT* ctx); + +void pj_load_ini(PJ_CONTEXT* ctx); /* classic public API */ #include "proj_api.h" diff --git a/src/transformations/hgridshift.cpp b/src/transformations/hgridshift.cpp index e9983df6..24da4dde 100644 --- a/src/transformations/hgridshift.cpp +++ b/src/transformations/hgridshift.cpp @@ -17,6 +17,7 @@ struct hgridshiftData { double t_final = 0; double t_epoch = 0; ListOfHGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace @@ -25,6 +26,14 @@ static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -40,6 +49,14 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_hgrid_init(P, "grids"); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -94,10 +111,19 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct hgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + PJ *TRANSFORMATION(hgridshift,0) { auto Q = new hgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; P->fwd4d = forward_4d; P->inv4d = reverse_4d; @@ -114,9 +140,9 @@ PJ *TRANSFORMATION(hgridshift,0) { return destructor (P, PJD_ERR_NO_ARGS); } - /* TODO: Refactor into shared function that can be used */ - /* by both vgridshift and hgridshift */ - if (pj_param(P->ctx, P->params, "tt_final").i) { + /* TODO: Refactor into shared function that can be used */ + /* by both vgridshift and hgridshift */ + if (pj_param(P->ctx, P->params, "tt_final").i) { Q->t_final = pj_param (P->ctx, P->params, "dt_final").f; if (Q->t_final == 0) { /* a number wasn't passed to +t_final, let's see if it was "now" */ @@ -131,16 +157,21 @@ PJ *TRANSFORMATION(hgridshift,0) { } } - if (pj_param(P->ctx, P->params, "tt_epoch").i) + if (pj_param(P->ctx, P->params, "tt_epoch").i) Q->t_epoch = pj_param (P->ctx, P->params, "dt_epoch").f; - Q->grids = proj_hgrid_init(P, "grids"); - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "hgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; } + else { + Q->grids = proj_hgrid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "hgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + } return P; } diff --git a/src/transformations/vgridshift.cpp b/src/transformations/vgridshift.cpp index b964f45b..3e7a015e 100644 --- a/src/transformations/vgridshift.cpp +++ b/src/transformations/vgridshift.cpp @@ -18,14 +18,51 @@ struct vgridshiftData { double t_epoch = 0; double forward_multiplier = 0; ListOfVGrids grids{}; + bool defer_grid_opening = false; }; } // anonymous namespace +static void deal_with_vertcon_gtx_hack(PJ *P) +{ + struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; + // The .gtx VERTCON files stored millimetres, but the .tif files + // are in metres. + if( Q->forward_multiplier != 0.001 ) { + return; + } + const char* gridname = pj_param(P->ctx, P->params, "sgrids").s; + if( !gridname ) { + return; + } + if( strcmp(gridname, "vertconw.gtx") != 0 && + strcmp(gridname, "vertconc.gtx") != 0 && + strcmp(gridname, "vertcone.gtx") != 0 ) { + return; + } + if( Q->grids.empty() ) { + return; + } + const auto& grids = Q->grids[0]->grids(); + if( !grids.empty() && + grids[0]->name().find(".tif") != std::string::npos ) { + Q->forward_multiplier = 1.0; + } +} + static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { struct vgridshiftData *Q = (struct vgridshiftData *) P->opaque; PJ_COORD point = {{0,0,0,0}}; point.lpz = lpz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().xyz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -41,6 +78,15 @@ static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { PJ_COORD point = {{0,0,0,0}}; point.xyz = xyz; + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_vgrid_init(P, "grids"); + deal_with_vertcon_gtx_hack(P); + if ( proj_errno(P) ) { + return proj_coord_error().lpz; + } + } + if (!Q->grids.empty()) { /* Only try the gridshift if at least one grid is loaded, * otherwise just pass the coordinate through unchanged. */ @@ -96,10 +142,20 @@ static PJ *destructor (PJ *P, int errlev) { return pj_default_destructor(P, errlev); } +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct vgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + + PJ *TRANSFORMATION(vgridshift,0) { auto Q = new vgridshiftData; P->opaque = (void *) Q; P->destructor = destructor; + P->reassign_context = reassign_context; if (!pj_param(P->ctx, P->params, "tgrids").i) { proj_log_error(P, "vgridshift: +grids parameter missing."); @@ -132,13 +188,18 @@ PJ *TRANSFORMATION(vgridshift,0) { Q->forward_multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; } - /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ - Q->grids = proj_vgrid_init(P, "grids"); - - /* Was gridlist compiled properly? */ - if ( proj_errno(P) ) { - proj_log_error(P, "vgridshift: could not find required grid(s)."); - return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + /* Build gridlist. P->vgridlist_geoid can be empty if +grids only ask for optional grids. */ + Q->grids = proj_vgrid_init(P, "grids"); + + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "vgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } } P->fwd4d = forward_4d; diff --git a/src/transformations/xyzgridshift.cpp b/src/transformations/xyzgridshift.cpp new file mode 100644 index 00000000..a76f3255 --- /dev/null +++ b/src/transformations/xyzgridshift.cpp @@ -0,0 +1,353 @@ +/****************************************************************************** + * Project: PROJ + * Purpose: Geocentric translation using a grid + * Author: Even Rouault, <even.rouault at spatialys.com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault, <even.rouault at spatialys.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#define PJ_LIB__ + +#include <errno.h> +#include <string.h> +#include <stddef.h> +#include <time.h> + +#include "proj_internal.h" +#include "grids.hpp" + +#include <algorithm> + +PROJ_HEAD(xyzgridshift, "Geocentric grid shift"); + +using namespace NS_PROJ; + +namespace { // anonymous namespace +struct xyzgridshiftData { + PJ *cart = nullptr; + bool grid_ref_is_input = true; + ListOfGenericGrids grids{}; + bool defer_grid_opening = false; + double multiplier = 1.0; +}; +} // anonymous namespace + + +// --------------------------------------------------------------------------- + +static const GenericShiftGrid* findGrid(const ListOfGenericGrids& grids, + const PJ_LP& input) +{ + for( const auto& gridset: grids ) + { + auto grid = gridset->gridAt(input.lam, input.phi); + if( grid ) + return grid; + } + return nullptr; +} + +// --------------------------------------------------------------------------- + +static bool get_grid_values(PJ* P, + xyzgridshiftData* Q, + const PJ_LP& lp, + double& dx, + double& dy, + double& dz) +{ + if ( Q->defer_grid_opening ) { + Q->defer_grid_opening = false; + Q->grids = proj_generic_grid_init(P, "grids"); + if ( proj_errno(P) ) { + return false; + } + } + + auto grid = findGrid(Q->grids, lp); + if( !grid ) { + return false; + } + if( grid->isNullGrid() ) { + dx = 0; + dy = 0; + dz = 0; + return true; + } + const auto samplesPerPixel = grid->samplesPerPixel(); + if( samplesPerPixel < 3 ) { + proj_log_error(P, "xyzgridshift: grid has not enough samples"); + return false; + } + int sampleX = 0; + int sampleY = 1; + int sampleZ = 2; + for( int i = 0; i < samplesPerPixel; i++ ) + { + const auto desc = grid->description(i); + if( desc == "x_translation") { + sampleX = i; + } else if( desc == "y_translation") { + sampleY = i; + } else if( desc == "z_translation") { + sampleZ = i; + } + } + const auto unit = grid->unit(sampleX); + if( !unit.empty() && unit != "metre" ) { + proj_log_error(P, "xyzgridshift: Only unit=metre currently handled"); + return false; + } + + const auto& extent = grid->extentAndRes(); + double grid_x = (lp.lam - extent.westLon) / extent.resLon; + double grid_y = (lp.phi - extent.southLat) / extent.resLat; + int ix = static_cast<int>(grid_x); + int iy = static_cast<int>(grid_y); + int ix2 = std::min(ix + 1, grid->width() - 1); + int iy2 = std::min(iy + 1, grid->height() - 1); + + float dx1, dy1, dz1; + if( !grid->valueAt(ix, iy, sampleX, dx1) || + !grid->valueAt(ix, iy, sampleY, dy1) || + !grid->valueAt(ix, iy, sampleZ, dz1) ) { + return false; + } + + float dx2, dy2, dz2; + if( !grid->valueAt(ix2, iy, sampleX, dx2) || + !grid->valueAt(ix2, iy, sampleY, dy2) || + !grid->valueAt(ix2, iy, sampleZ, dz2) ) { + return false; + } + + float dx3, dy3, dz3; + if( !grid->valueAt(ix, iy2, sampleX, dx3) || + !grid->valueAt(ix, iy2, sampleY, dy3) || + !grid->valueAt(ix, iy2, sampleZ, dz3) ) { + return false; + } + + float dx4, dy4, dz4; + if( !grid->valueAt(ix2, iy2, sampleX, dx4) || + !grid->valueAt(ix2, iy2, sampleY, dy4) || + !grid->valueAt(ix2, iy2, sampleZ, dz4) ) { + return false; + } + + double frct_lam = grid_x - ix; + double frct_phi = grid_y - iy; + double m10 = frct_lam; + double m11 = m10; + double m01 = 1. - frct_lam; + double m00 = m01; + m11 *= frct_phi; + m01 *= frct_phi; + frct_phi = 1. - frct_phi; + m00 *= frct_phi; + m10 *= frct_phi; + dx = (m00 * dx1 + m10 * dx2 + m01 * dx3 + m11 * dx4) * Q->multiplier; + dy = (m00 * dy1 + m10 * dy2 + m01 * dy3 + m11 * dy4) * Q->multiplier; + dz = (m00 * dz1 + m10 * dz2 + m01 * dz3 + m11 * dz4) * Q->multiplier; + return true; +} + +// --------------------------------------------------------------------------- + +#define SQUARE(x) ((x)*(x)) + +// --------------------------------------------------------------------------- + +static PJ_COORD iterative_adjustment(PJ* P, + xyzgridshiftData* Q, + const PJ_COORD& pointInit, + double factor) +{ + PJ_COORD point = pointInit; + for(int i = 0; i < 10; i++) { + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + + dx *= factor; + dy *= factor; + dz *= factor; + + const double err = SQUARE((point.xyz.x - pointInit.xyz.x) - dx) + + SQUARE((point.xyz.y - pointInit.xyz.y) - dy) + + SQUARE((point.xyz.z - pointInit.xyz.z) - dz); + + point.xyz.x = pointInit.xyz.x + dx; + point.xyz.y = pointInit.xyz.y + dy; + point.xyz.z = pointInit.xyz.z + dz; + if( err < 1e-10 ) { + break; + } + } + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_COORD direct_adjustment(PJ* P, + xyzgridshiftData* Q, + PJ_COORD point, + double factor) +{ + PJ_COORD geodetic; + geodetic.lpz = pj_inv3d(point.xyz, Q->cart); + + double dx, dy, dz; + if( !get_grid_values(P, Q, geodetic.lp, dx, dy, dz) ) { + return proj_coord_error(); + } + point.xyz.x += factor * dx; + point.xyz.y += factor * dy; + point.xyz.z += factor * dz; + return point; +} + +// --------------------------------------------------------------------------- + +static PJ_XYZ forward_3d(PJ_LPZ lpz, PJ *P) { + auto Q = static_cast<xyzgridshiftData*>(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.lpz = lpz; + + if( Q->grid_ref_is_input ) { + point = direct_adjustment(P, Q, point, 1.0); + } + else { + point = iterative_adjustment(P, Q, point, 1.0); + } + + return point.xyz; +} + + +static PJ_LPZ reverse_3d(PJ_XYZ xyz, PJ *P) { + auto Q = static_cast<xyzgridshiftData*>(P->opaque); + PJ_COORD point = {{0,0,0,0}}; + point.xyz = xyz; + + if( Q->grid_ref_is_input ) { + point = iterative_adjustment(P, Q, point, -1.0); + } + else { + point = direct_adjustment(P, Q, point, -1.0); + } + + return point.lpz; +} + +static PJ *destructor (PJ *P, int errlev) { + if (nullptr==P) + return nullptr; + + auto Q = static_cast<struct xyzgridshiftData*>(P->opaque); + if( Q ) + { + if (Q->cart) + Q->cart->destructor (Q->cart, errlev); + delete Q; + } + P->opaque = nullptr; + + return pj_default_destructor(P, errlev); +} + +static void reassign_context( PJ* P, PJ_CONTEXT* ctx ) +{ + auto Q = (struct xyzgridshiftData *) P->opaque; + for( auto& grid: Q->grids ) { + grid->reassign_context(ctx); + } +} + + +PJ *TRANSFORMATION(xyzgridshift,0) { + auto Q = new xyzgridshiftData; + P->opaque = (void *) Q; + P->destructor = destructor; + P->reassign_context = reassign_context; + + P->fwd4d = nullptr; + P->inv4d = nullptr; + P->fwd3d = forward_3d; + P->inv3d = reverse_3d; + P->fwd = nullptr; + P->inv = nullptr; + + P->left = PJ_IO_UNITS_CARTESIAN; + P->right = PJ_IO_UNITS_CARTESIAN; + + // Pass a dummy ellipsoid definition that will be overridden just afterwards + Q->cart = proj_create(P->ctx, "+proj=cart +a=1"); + if (Q->cart == nullptr) + return destructor(P, ENOMEM); + + /* inherit ellipsoid definition from P to Q->cart */ + pj_inherit_ellipsoid_def (P, Q->cart); + + const char* grid_ref = pj_param (P->ctx, P->params, "sgrid_ref").s; + if( grid_ref ) { + if (strcmp(grid_ref, "input_crs") == 0 ) { + // default + } else if (strcmp(grid_ref, "output_crs") == 0 ) { + // Convention use for example for NTF->RGF93 grid that contains + // delta x,y,z from NTF to RGF93, but the grid itself is referenced + // in RGF93 + Q->grid_ref_is_input = false; + } else { + proj_log_error(P, "xyzgridshift: unusupported value for grid_ref"); + return destructor (P, PJD_ERR_NO_ARGS); + } + } + + if (0==pj_param(P->ctx, P->params, "tgrids").i) { + proj_log_error(P, "xyzgridshift: +grids parameter missing."); + return destructor (P, PJD_ERR_NO_ARGS); + } + + /* multiplier for delta x,y,z */ + if (pj_param(P->ctx, P->params, "tmultiplier").i) { + Q->multiplier = pj_param(P->ctx, P->params, "dmultiplier").f; + } + + if( P->ctx->defer_grid_opening ) { + Q->defer_grid_opening = true; + } + else { + Q->grids = proj_generic_grid_init(P, "grids"); + /* Was gridlist compiled properly? */ + if ( proj_errno(P) ) { + proj_log_error(P, "xyzgridshift: could not find required grid(s)."); + return destructor(P, PJD_ERR_FAILED_TO_LOAD_GRID); + } + } + + return P; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ae721d46..a7aac755 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -10,6 +10,7 @@ proj_add_gie_test("GDA" "gie/GDA.gie") proj_add_gie_test("4D-API-cs2cs-style" "gie/4D-API_cs2cs-style.gie") proj_add_gie_test("DHDN_ETRS89" "gie/DHDN_ETRS89.gie") proj_add_gie_test("Unitconvert" "gie/unitconvert.gie") +proj_add_gie_test("geotiff_grids" "gie/geotiff_grids.gie") # GIGS tests. Uncommented tests are expected to fail due to issues with # various projections. Should be investigated further and fixed. diff --git a/test/cli/ntv2_out.dist b/test/cli/ntv2_out.dist index 9a97f9cf..650a69d8 100644 --- a/test/cli/ntv2_out.dist +++ b/test/cli/ntv2_out.dist @@ -9,3 +9,6 @@ Try with NTv2 and NTv1 together ... falls back to NTv1 99d00'00.000"W 65d00'00.000"N 0.0 99d0'1.5885"W 65d0'1.3482"N 0.000 111d00'00.000"W 46d00'00.000"N 0.0 111d0'3.1897"W 45d59'59.7489"N 0.000 111d00'00.000"W 47d30'00.000"N 0.0 111d0'2.7989"W 47d29'59.9896"N 0.000 +############################################################## +Switching between NTv2 subgrids +-112.5839956 49.4914451 0 -112.58307487 49.49145197 0.00000000 diff --git a/test/cli/test27 b/test/cli/test27 index bfc1cb0a..5825ed80 100755 --- a/test/cli/test27 +++ b/test/cli/test27 @@ -34,7 +34,7 @@ echo "Running ${0} using ${EXE}:" echo "============================================" OUT=proj_out27 -INIT_FILE=${PROJ_LIB}/nad27 +INIT_FILE=nad27 # echo "doing tests into file ${OUT}, please wait" # diff --git a/test/cli/test83 b/test/cli/test83 index cfb1365e..8c1293d0 100755 --- a/test/cli/test83 +++ b/test/cli/test83 @@ -35,7 +35,7 @@ echo "Running ${0} using ${EXE}:" echo "============================================" OUT=proj_out83 -INIT_FILE=${PROJ_LIB}/nad83 +INIT_FILE=nad83 # echo "doing tests into file ${OUT}, please wait" # diff --git a/test/cli/testdatumfile b/test/cli/testdatumfile index 29a40876..9bd12ce4 100755 --- a/test/cli/testdatumfile +++ b/test/cli/testdatumfile @@ -27,7 +27,11 @@ echo "Running ${0} using ${EXE}:" echo "============================================" mkdir "dir with \" space" -cp ${PROJ_LIB}/conus "dir with \" space/myconus" +if test -f "${PROJ_LIB}/conus"; then + cp "${PROJ_LIB}/conus" "dir with \" space/myconus" +else + cp "`dirname $0`/../../data/conus" "dir with \" space/myconus" +fi OUT=td_out #EXE=../src/cs2cs diff --git a/test/cli/testntv2 b/test/cli/testntv2 index 73371dbe..2a31304e 100755 --- a/test/cli/testntv2 +++ b/test/cli/testntv2 @@ -52,6 +52,14 @@ $EXE +proj=latlong +ellps=clrk66 +nadgrids=ntv2_0.gsb,ntv1_can.dat,conus \ 111d00'00.000"W 46d00'00.000"N 0.0 111d00'00.000"W 47d30'00.000"N 0.0 EOF + +echo "##############################################################" >> ${OUT} +echo Switching between NTv2 subgrids >> ${OUT} +# Initial guess is in ALraymnd, going to parent CAwest afterwards +$EXE +proj=latlong +datum=NAD83 +to +proj=latlong +ellps=clrk66 +nadgrids=ntv2_0.gsb -E -d 8 >>${OUT} <<EOF +-112.5839956 49.4914451 0 +EOF + # ############################################################################## # Done! diff --git a/test/gie/4D-API_cs2cs-style.gie b/test/gie/4D-API_cs2cs-style.gie index 8d541823..3e4b9d2c 100644 --- a/test/gie/4D-API_cs2cs-style.gie +++ b/test/gie/4D-API_cs2cs-style.gie @@ -442,7 +442,6 @@ Test bugfix of https://github.com/OSGeo/proj.4/issues/1002 ------------------------------------------------------------------------------- operation +proj=latlong +ellps=WGS84 +geoidgrids=tests/test_nodata.gtx ------------------------------------------------------------------------------- -ignore pjd_err_failed_to_load_grid accept 4.05 52.1 0 expect 4.05 52.1 -10 ------------------------------------------------------------------------------- @@ -452,7 +451,6 @@ Test null grid with vgridshift ------------------------------------------------------------------------------- operation proj=vgridshift grids=tests/test_nodata.gtx,null ellps=GRS80 ------------------------------------------------------------------------------- -ignore pjd_err_failed_to_load_grid accept 4.05 52.1 0 expect 4.05 52.1 -10 diff --git a/test/gie/Makefile.am b/test/gie/Makefile.am index 44facd87..ff333a14 100644 --- a/test/gie/Makefile.am +++ b/test/gie/Makefile.am @@ -9,7 +9,8 @@ EXTRA_DIST = 4D-API_cs2cs-style.gie \ ellipsoid.gie \ more_builtins.gie \ unitconvert.gie \ - DHDN_ETRS89.gie + DHDN_ETRS89.gie \ + geotiff_grids.gie PROJ_LIB ?= ../../data @@ -40,4 +41,7 @@ unitconvert: unitconvert.gie DHDN_ETRS89: DHDN_ETRS89.gie PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< -check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 +geotiff_grids: geotiff_grids.gie + PROJ_LIB=$(PROJ_LIB) $(GIEEXE) $< + +check-local: 4D-API-cs2cs-style GDA axisswap builtins deformation ellipsoid more_builtins unitconvert DHDN_ETRS89 geotiff_grids diff --git a/test/gie/geotiff_grids.gie b/test/gie/geotiff_grids.gie new file mode 100644 index 00000000..920fcf28 --- /dev/null +++ b/test/gie/geotiff_grids.gie @@ -0,0 +1,306 @@ + +------------------------------------------------------------------------------- +=============================================================================== +Test GeoTIFF grids +=============================================================================== + +<gie> + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_pixelispoint.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_pixelisarea.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_deflate.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_deflate_floatingpointpredictor.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint16.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint16_with_scale_offset.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_int16.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_int32.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_uint32.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_float64.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +# The overview should be ignored +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_with_overview.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_in_second_channel.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigtiff.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigendian.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bigendian_bigtiff.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bottomup_with_scale.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_bottomup_with_matrix.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_with_subgrid.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.5 52.5 0 +expect 4.5 52.5 11.5 + +# In subgrid +accept 5.5 53.5 0 +expect 5.5 53.5 110.0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_nodata.tif +multiplier=1 +------------------------------------------------------------------------------- +accept 4.05 52.1 0 +expect 4.05 52.1 10 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_invalid_channel_type.tif +multiplier=1 +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=vgridshift +grids=tests/test_vgrid_unsupported_byte.tif +multiplier=1 +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + + + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_separate.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_strip.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_tiled.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_tiled_separate.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_positive_west.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_lon_shift_first.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_radian.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_degree.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +# The overview should be ignored +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_overview.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_extra_ifd_with_other_info.tif +------------------------------------------------------------------------------- +tolerance 2 mm +accept 4.5 52.5 0 +expect 5.875 55.375 0 +------------------------------------------------------------------------------- + +# Subset of NTv2_0.gsb +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_subgrid.tif +------------------------------------------------------------------------------- +# In subgrid ALbanff, of parent CAwest +accept -115.5416667 51.1666667 0 +expect -115.5427092888 51.1666899972 0 + +# In subgrid ONtronto, of parent CAeast +accept -80.5041667 44.5458333 0 +expect -80.50401615833 44.5458827236 0 +------------------------------------------------------------------------------- + +# Subset of NTv2_0.gsb +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_hgrid_with_subgrid_no_grid_name.tif +------------------------------------------------------------------------------- +# In subgrid ALbanff, of parent CAwest +accept -115.5416667 51.1666667 0 +expect -115.5427092888 51.1666899972 0 + +# In subgrid ONtronto, of parent CAeast +accept -80.5041667 44.5458333 0 +expect -80.50401615833 44.5458827236 0 +------------------------------------------------------------------------------- + +------------------------------------------------------------------------------- +operation +proj=hgridshift +grids=tests/test_vgrid.tif +------------------------------------------------------------------------------- +expect failure errno failed_to_load_grid +------------------------------------------------------------------------------- + + +# IGNF:LAMBE to IGNF:LAMB93 using xyzgridshift operation +------------------------------------------------------------------------------- +operation +proj=pipeline + +step +inv +proj=lcc +lat_1=46.8 +lat_0=46.8 +lon_0=0 + +k_0=0.99987742 +x_0=600000 +y_0=2200000 +ellps=clrk80ign +pm=paris + +step +proj=push +v_3 + +step +proj=cart +ellps=clrk80ign + +step +proj=xyzgridshift +grids=tests/subset_of_gr3df97a.tif +grid_ref=output_crs +ellps=GRS80 + +step +proj=cart +ellps=GRS80 +inv + +step +proj=pop +v_3 + +step +proj=lcc +lat_0=46.5 +lon_0=3 +lat_1=49 +lat_2=44 + +x_0=700000 +y_0=6600000 +ellps=GRS80 +------------------------------------------------------------------------------- +tolerance 1 mm + +accept 814149.529 1887019.768 0 +expect 860690.804 6319036.849 0 +# If using ntf_r93.gsb, one gets: 860690.805 6319036.850 + +roundtrip 1 +------------------------------------------------------------------------------- + + +</gie> diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 841d72b3..0dcafc85 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -147,3 +147,23 @@ target_link_libraries(gie_self_tests add_test(NAME gie_self_tests COMMAND gie_self_tests) set_property(TEST gie_self_tests PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data") + + +add_executable(test_network + main.cpp + test_network.cpp) +if(CURL_FOUND) + include_directories(${CURL_INCLUDE_DIR}) + target_link_libraries(test_network ${CURL_LIBRARY}) +endif() +target_link_libraries(test_network + GTest::gtest + ${PROJ_LIBRARIES}) +add_test(NAME test_network COMMAND test_network) +if(MSVC) + set_property(TEST test_network + PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data\\;${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") +else() + set_property(TEST test_network + PROPERTY ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data:${PROJECT_SOURCE_DIR}/data;PROJ_SOURCE_DATA=${PROJECT_SOURCE_DIR}/data") +endif() diff --git a/test/unit/Makefile.am b/test/unit/Makefile.am index 57a03ca8..ce11ae4e 100644 --- a/test/unit/Makefile.am +++ b/test/unit/Makefile.am @@ -17,6 +17,7 @@ noinst_PROGRAMS += proj_context_test noinst_PROGRAMS += test_cpp_api noinst_PROGRAMS += gie_self_tests noinst_PROGRAMS += include_proj_h_from_c +noinst_PROGRAMS += test_network pj_transform_test_SOURCES = pj_transform_test.cpp main.cpp pj_transform_test_LDADD = ../../src/libproj.la @GTEST_LIBS@ @@ -62,4 +63,11 @@ gie_self_tests-check: gie_self_tests include_proj_h_from_c_SOURCES = include_proj_h_from_c.c -check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check +test_network_SOURCES = test_network.cpp main.cpp +test_network_CXXFLAGS = @CURL_CFLAGS@ @CURL_ENABLED_FLAGS@ +test_network_LDADD = ../../src/libproj.la @GTEST_LIBS@ @CURL_LIBS@ + +test_network-check: test_network + PROJ_LIB=$(PROJ_LIB) PROJ_SOURCE_DATA=$(PROJ_LIB) ./test_network + +check-local: pj_transform_test-check pj_phi2_test-check proj_errno_string_test-check proj_angular_io_test-check proj_context_test-check test_cpp_api-check gie_self_tests-check test_network-check diff --git a/test/unit/pj_transform_test.cpp b/test/unit/pj_transform_test.cpp index b3a061b4..5ca8dcce 100644 --- a/test/unit/pj_transform_test.cpp +++ b/test/unit/pj_transform_test.cpp @@ -614,4 +614,95 @@ TEST(proj_api_h, pj_set_finder) { pj_set_finder(nullptr); } +// --------------------------------------------------------------------------- + +TEST(proj_api_h, default_fileapi) { + auto ctx = pj_ctx_alloc(); + auto fid = pj_open_lib(ctx, "proj.db", "rb"); + ASSERT_NE(fid, nullptr); + char header[6]; + ASSERT_EQ(pj_ctx_fread(ctx, header, 1, 6, fid), 6U); + ASSERT_TRUE(memcmp(header, "SQLite", 6) == 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 6); + ASSERT_EQ(pj_ctx_fseek(ctx, fid, 0, SEEK_SET), 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 0); + pj_ctx_fclose(ctx, fid); + pj_ctx_free(ctx); +} + +// --------------------------------------------------------------------------- + +struct Spy { + bool gotInMyFOpen = false; + bool gotInMyFRead = false; + bool gotInMyFSeek = false; + bool gotInMyFTell = false; + bool gotInMyFClose = false; +}; + +struct MyFile { + FILE *fp; + Spy *spy; +}; + +static PAFile myFOpen(projCtx ctx, const char *filename, const char *access) { + FILE *fp = fopen(filename, access); + if (!fp) + return nullptr; + MyFile *myF = new MyFile; + myF->spy = (Spy *)pj_ctx_get_app_data(ctx); + myF->spy->gotInMyFOpen = true; + myF->fp = fp; + return reinterpret_cast<PAFile>(myF); +} + +static size_t myFRead(void *buffer, size_t size, size_t nmemb, PAFile file) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFRead = true; + return fread(buffer, size, nmemb, myF->fp); +} + +static int myFSeek(PAFile file, long offset, int whence) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFSeek = true; + return fseek(myF->fp, offset, whence); +} + +static long myFTell(PAFile file) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFTell = true; + return ftell(myF->fp); +} + +static void myFClose(PAFile file) { + MyFile *myF = reinterpret_cast<MyFile *>(file); + myF->spy->gotInMyFClose = true; + fclose(myF->fp); + delete myF; +} + +TEST(proj_api_h, custom_fileapi) { + auto ctx = pj_ctx_alloc(); + Spy spy; + pj_ctx_set_app_data(ctx, &spy); + projFileAPI myAPI = {myFOpen, myFRead, myFSeek, myFTell, myFClose}; + pj_ctx_set_fileapi(ctx, &myAPI); + EXPECT_EQ(pj_ctx_get_fileapi(ctx), &myAPI); + auto fid = pj_open_lib(ctx, "proj.db", "rb"); + ASSERT_NE(fid, nullptr); + char header[6]; + ASSERT_EQ(pj_ctx_fread(ctx, header, 1, 6, fid), 6U); + ASSERT_TRUE(memcmp(header, "SQLite", 6) == 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 6); + ASSERT_EQ(pj_ctx_fseek(ctx, fid, 0, SEEK_SET), 0); + ASSERT_EQ(pj_ctx_ftell(ctx, fid), 0); + pj_ctx_fclose(ctx, fid); + pj_ctx_free(ctx); + EXPECT_TRUE(spy.gotInMyFOpen); + EXPECT_TRUE(spy.gotInMyFRead); + EXPECT_TRUE(spy.gotInMyFSeek); + EXPECT_TRUE(spy.gotInMyFTell); + EXPECT_TRUE(spy.gotInMyFClose); +} + } // namespace diff --git a/test/unit/test_factory.cpp b/test/unit/test_factory.cpp index 93b2ef34..e342dad9 100644 --- a/test/unit/test_factory.cpp +++ b/test/unit/test_factory.cpp @@ -1617,34 +1617,34 @@ class FactoryWithTmpDatabase : public ::testing::Test { DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {std::make_pair(std::string("NS_PIVOT"), - std::string("PIVOT"))}); + false, false, {std::make_pair(std::string("NS_PIVOT"), + std::string("PIVOT"))}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {std::make_pair(std::string("NS_PIVOT"), - std::string("NOT_EXISTING"))}); + false, false, {std::make_pair(std::string("NS_PIVOT"), + std::string("NOT_EXISTING"))}); EXPECT_EQ(res.size(), 0U); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, + false, false, {std::make_pair(std::string("BAD_NS"), std::string("PIVOT"))}); EXPECT_EQ(res.size(), 0U); res = factoryOTHER->createFromCRSCodesWithIntermediates( "NS_TARGET", "TARGET", "NS_SOURCE", "SOURCE", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); @@ -1654,7 +1654,7 @@ class FactoryWithTmpDatabase : public ::testing::Test { DatabaseContext::create(m_ctxt), std::string()); auto res = factory->createFromCRSCodesWithIntermediates( "NS_SOURCE", "SOURCE", "NS_TARGET", "TARGET", false, false, - false, {}); + false, false, {}); EXPECT_EQ(res.size(), 1U); EXPECT_TRUE(res.empty() || nn_dynamic_pointer_cast<ConcatenatedOperation>(res[0])); @@ -1833,7 +1833,7 @@ TEST(factory, AuthorityFactory_createFromCoordinateReferenceSystemCodes) { { // Test removal of superseded transform auto list = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4179", "EPSG", "4258", false, false, true); + "EPSG", "4179", "EPSG", "4258", false, false, false, true); ASSERT_EQ(list.size(), 2U); // Romania has a larger area than Poland (given our approx formula) EXPECT_EQ(list[0]->getEPSGCode(), 15994); // Romania - 3m @@ -1851,12 +1851,12 @@ TEST( { auto res = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "32631", false, false, false); + "EPSG", "4326", "EPSG", "32631", false, false, false, false); ASSERT_EQ(res.size(), 1U); } { auto res = factory->createFromCoordinateReferenceSystemCodes( - "EPSG", "4209", "EPSG", "4326", false, false, false); + "EPSG", "4209", "EPSG", "4326", false, false, false, false); EXPECT_TRUE(!res.empty()); for (const auto &conv : res) { EXPECT_TRUE(conv->sourceCRS()->getEPSGCode() == 4209); @@ -1889,7 +1889,8 @@ TEST_F(FactoryWithTmpDatabase, DatabaseContext::create(m_ctxt), std::string()); { auto res = factoryGeneral->createFromCoordinateReferenceSystemCodes( - "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false); + "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false, + false); ASSERT_EQ(res.size(), 1U); } @@ -1897,7 +1898,8 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory::create(DatabaseContext::create(m_ctxt), "EPSG"); { auto res = factoryEPSG->createFromCoordinateReferenceSystemCodes( - "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false); + "OTHER", "OTHER_4326", "OTHER", "OTHER_32631", false, false, false, + false); ASSERT_EQ(res.size(), 1U); } @@ -1919,17 +1921,17 @@ TEST_F(FactoryWithTmpDatabase, << last_error(); { auto res = factoryGeneral->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); } { auto res = factoryEPSG->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 0U); } { auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false); + "EPSG", "4326", "OTHER", "OTHER_4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); } } @@ -1982,7 +1984,7 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 3U); EXPECT_EQ(*(res[0]->name()->description()), "TRANSFORMATION_1M"); EXPECT_EQ(*(res[1]->name()->description()), "TRANSFORMATION_10M"); @@ -2001,7 +2003,7 @@ TEST_F( auto factory = AuthorityFactory::create(DatabaseContext::create(m_ctxt), std::string()); auto res = factory->createFromCRSCodesWithIntermediates( - "EPSG", "4326", "EPSG", "4326", false, false, false, {}); + "EPSG", "4326", "EPSG", "4326", false, false, false, false, {}); EXPECT_EQ(res.size(), 0U); } @@ -2085,7 +2087,7 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory_proj_based_transformation) { auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); EXPECT_EQ(res[0]->nameStr(), "My PROJ string based op"); EXPECT_EQ(res[0]->exportToPROJString(PROJStringFormatter::create().get()), @@ -2146,7 +2148,7 @@ TEST_F(FactoryWithTmpDatabase, AuthorityFactory_wkt_based_transformation) { auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); auto res = factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false); + "EPSG", "4326", "EPSG", "4326", false, false, false, false); ASSERT_EQ(res.size(), 1U); EXPECT_EQ(res[0]->nameStr(), "My WKT string based op"); EXPECT_EQ(res[0]->exportToPROJString(PROJStringFormatter::create().get()), @@ -2180,9 +2182,10 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); - EXPECT_THROW(factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false), - FactoryException); + EXPECT_THROW( + factoryOTHER->createFromCoordinateReferenceSystemCodes( + "EPSG", "4326", "EPSG", "4326", false, false, false, false), + FactoryException); } // --------------------------------------------------------------------------- @@ -2207,9 +2210,10 @@ TEST_F(FactoryWithTmpDatabase, auto factoryOTHER = AuthorityFactory::create(DatabaseContext::create(m_ctxt), "OTHER"); - EXPECT_THROW(factoryOTHER->createFromCoordinateReferenceSystemCodes( - "EPSG", "4326", "EPSG", "4326", false, false, false), - FactoryException); + EXPECT_THROW( + factoryOTHER->createFromCoordinateReferenceSystemCodes( + "EPSG", "4326", "EPSG", "4326", false, false, false, false), + FactoryException); } // --------------------------------------------------------------------------- @@ -2262,7 +2266,7 @@ TEST_F(FactoryWithTmpDatabase, lookForGridInfo) { bool openLicense = false; bool gridAvailable = false; EXPECT_TRUE(DatabaseContext::create(m_ctxt)->lookForGridInfo( - "PROJ_fake_grid", fullFilename, packageName, url, directDownload, + "PROJ_fake_grid", false, fullFilename, packageName, url, directDownload, openLicense, gridAvailable)); EXPECT_TRUE(fullFilename.empty()); EXPECT_TRUE(packageName.empty()); diff --git a/test/unit/test_network.cpp b/test/unit/test_network.cpp new file mode 100644 index 00000000..5cd32f68 --- /dev/null +++ b/test/unit/test_network.cpp @@ -0,0 +1,879 @@ +/****************************************************************************** + * + * Project: PROJ + * Purpose: Test networking + * Author: Even Rouault <even dot rouault at spatialys dot com> + * + ****************************************************************************** + * Copyright (c) 2019, Even Rouault <even dot rouault at spatialys dot com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + ****************************************************************************/ + +#include "gtest_include.h" + +#include <memory> +#include <stdio.h> +#include <stdlib.h> + +#include "proj_internal.h" +#include <proj.h> + +#ifdef CURL_ENABLED +#include <curl/curl.h> +#endif + +namespace { + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +static bool networkAccessOK = false; + +static size_t noop_curl_write_func(void *, size_t, size_t nmemb, void *) { + return nmemb; +} + +TEST(networking, initial_check) { + CURL *hCurlHandle = curl_easy_init(); + if (!hCurlHandle) + return; + curl_easy_setopt(hCurlHandle, CURLOPT_URL, + "https://cdn.proj.org/ntf_r93.tif"); + + curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, "0-1"); + curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, noop_curl_write_func); + + curl_easy_perform(hCurlHandle); + + long response_code = 0; + curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code); + + curl_easy_cleanup(hCurlHandle); + + networkAccessOK = (response_code == 206); + if (!networkAccessOK) { + fprintf(stderr, "network access not working"); + } +} + +#endif + +// --------------------------------------------------------------------------- + +static void silent_logger(void *, int, const char *) {} + +// --------------------------------------------------------------------------- + +TEST(networking, basic) { + const char *pipeline = + "+proj=pipeline " + "+step +proj=unitconvert +xy_in=deg +xy_out=rad " + "+step +proj=hgridshift +grids=https://cdn.proj.org/ntf_r93.tif " + "+step +proj=unitconvert +xy_in=rad +xy_out=deg"; + + // network access disabled by default + auto ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + auto P = proj_create(ctx, pipeline); + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + +#ifdef CURL_ENABLED + // enable through env variable + ctx = proj_context_create(); + putenv(const_cast<char *>("PROJ_NETWORK=ON")); + P = proj_create(ctx, pipeline); + if (networkAccessOK) { + ASSERT_NE(P, nullptr); + } + proj_destroy(P); + proj_context_destroy(ctx); + putenv(const_cast<char *>("PROJ_NETWORK=")); +#endif + + // still disabled + ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + P = proj_create(ctx, pipeline); + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + + // enable through API + ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + P = proj_create(ctx, pipeline); +#ifdef CURL_ENABLED + if (networkAccessOK) { + ASSERT_NE(P, nullptr); + } else { + ASSERT_EQ(P, nullptr); + proj_context_destroy(ctx); + return; + } + double lon = 2; + double lat = 49; + proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, sizeof(double), + 1, nullptr, 0, 0, nullptr, 0, 0); + EXPECT_NEAR(lon, 1.9992776848, 1e-10); + EXPECT_NEAR(lat, 48.9999322600, 1e-10); + + proj_destroy(P); +#else + ASSERT_EQ(P, nullptr); +#endif + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_invalid_resource) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + proj_log_func(ctx, nullptr, silent_logger); + auto P = proj_create( + ctx, "+proj=hgridshift +grids=https://i_do_not.exist/my.tif"); + proj_context_destroy(ctx); + ASSERT_EQ(P, nullptr); +} +#endif + +// --------------------------------------------------------------------------- + +struct Event { + virtual ~Event(); + std::string type{}; + PJ_CONTEXT *ctx = nullptr; +}; + +Event::~Event() = default; + +struct OpenEvent : public Event { + OpenEvent() { type = "OpenEvent"; } + + std::string url{}; + unsigned long long offset = 0; + size_t size_to_read = 0; + std::vector<unsigned char> response{}; + std::string errorMsg{}; + int file_id = 0; +}; + +struct CloseEvent : public Event { + CloseEvent() { type = "CloseEvent"; } + + int file_id = 0; +}; + +struct GetHeaderValueEvent : public Event { + GetHeaderValueEvent() { type = "GetHeaderValueEvent"; } + + int file_id = 0; + std::string key{}; + std::string value{}; +}; + +struct ReadRangeEvent : public Event { + ReadRangeEvent() { type = "ReadRangeEvent"; } + + unsigned long long offset = 0; + size_t size_to_read = 0; + std::vector<unsigned char> response{}; + std::string errorMsg{}; + int file_id = 0; +}; + +struct File {}; + +struct ExchangeWithCallback { + std::vector<std::unique_ptr<Event>> events{}; + size_t nextEvent = 0; + bool error = false; + std::map<int, PROJ_NETWORK_HANDLE *> mapIdToHandle{}; + + bool allConsumedAndNoError() const { + return nextEvent == events.size() && !error; + } +}; + +static PROJ_NETWORK_HANDLE *open_cbk(PJ_CONTEXT *ctx, const char *url, + unsigned long long offset, + size_t size_to_read, void *buffer, + size_t *out_size_read, + size_t error_string_max_size, + char *out_error_string, void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return nullptr; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to open(%s, %ld, %ld)\n", url, + (long)offset, (long)size_to_read); + exchange->error = true; + return nullptr; + } + auto openEvent = + dynamic_cast<OpenEvent *>(exchange->events[exchange->nextEvent].get()); + if (!openEvent) { + fprintf(stderr, "unexpected call to open(%s, %ld, %ld). " + "Was expecting a %s event\n", + url, (long)offset, (long)size_to_read, + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return nullptr; + } + exchange->nextEvent++; + if (openEvent->ctx != ctx || openEvent->url != url || + openEvent->offset != offset || + openEvent->size_to_read != size_to_read) { + fprintf(stderr, "wrong call to open(%s, %ld, %ld). Was expecting " + "open(%s, %ld, %ld)\n", + url, (long)offset, (long)size_to_read, openEvent->url.c_str(), + (long)openEvent->offset, (long)openEvent->size_to_read); + exchange->error = true; + return nullptr; + } + if (!openEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + openEvent->errorMsg.c_str()); + return nullptr; + } + + memcpy(buffer, openEvent->response.data(), openEvent->response.size()); + *out_size_read = openEvent->response.size(); + auto handle = reinterpret_cast<PROJ_NETWORK_HANDLE *>(new File()); + exchange->mapIdToHandle[openEvent->file_id] = handle; + return handle; +} + +static void close_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, + void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to close()\n"); + exchange->error = true; + return; + } + auto closeEvent = + dynamic_cast<CloseEvent *>(exchange->events[exchange->nextEvent].get()); + if (!closeEvent) { + fprintf(stderr, "unexpected call to close(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return; + } + if (closeEvent->ctx != ctx) { + fprintf(stderr, "close() called with bad context\n"); + exchange->error = true; + return; + } + if (exchange->mapIdToHandle[closeEvent->file_id] != handle) { + fprintf(stderr, "close() called with bad handle\n"); + exchange->error = true; + return; + } + exchange->nextEvent++; + delete reinterpret_cast<File *>(handle); +} + +static const char *get_header_value_cbk(PJ_CONTEXT *ctx, + PROJ_NETWORK_HANDLE *handle, + const char *header_name, + void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return nullptr; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to get_header_value()\n"); + exchange->error = true; + return nullptr; + } + auto getHeaderValueEvent = dynamic_cast<GetHeaderValueEvent *>( + exchange->events[exchange->nextEvent].get()); + if (!getHeaderValueEvent) { + fprintf(stderr, "unexpected call to get_header_value(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return nullptr; + } + if (getHeaderValueEvent->ctx != ctx) { + fprintf(stderr, "get_header_value() called with bad context\n"); + exchange->error = true; + return nullptr; + } + if (getHeaderValueEvent->key != header_name) { + fprintf(stderr, "wrong call to get_header_value(%s). Was expecting " + "get_header_value(%s)\n", + header_name, getHeaderValueEvent->key.c_str()); + exchange->error = true; + return nullptr; + } + if (exchange->mapIdToHandle[getHeaderValueEvent->file_id] != handle) { + fprintf(stderr, "get_header_value() called with bad handle\n"); + exchange->error = true; + return nullptr; + } + exchange->nextEvent++; + return getHeaderValueEvent->value.c_str(); +} + +static size_t read_range_cbk(PJ_CONTEXT *ctx, PROJ_NETWORK_HANDLE *handle, + unsigned long long offset, size_t size_to_read, + void *buffer, size_t error_string_max_size, + char *out_error_string, void *user_data) { + auto exchange = static_cast<ExchangeWithCallback *>(user_data); + if (exchange->error) + return 0; + if (exchange->nextEvent >= exchange->events.size()) { + fprintf(stderr, "unexpected call to read_range(%ld, %ld)\n", + (long)offset, (long)size_to_read); + exchange->error = true; + return 0; + } + auto readRangeEvent = dynamic_cast<ReadRangeEvent *>( + exchange->events[exchange->nextEvent].get()); + if (!readRangeEvent) { + fprintf(stderr, "unexpected call to read_range(). " + "Was expecting a %s event\n", + exchange->events[exchange->nextEvent]->type.c_str()); + exchange->error = true; + return 0; + } + if (exchange->mapIdToHandle[readRangeEvent->file_id] != handle) { + fprintf(stderr, "read_range() called with bad handle\n"); + exchange->error = true; + return 0; + } + if (readRangeEvent->ctx != ctx || readRangeEvent->offset != offset || + readRangeEvent->size_to_read != size_to_read) { + fprintf(stderr, "wrong call to read_range(%ld, %ld). Was expecting " + "read_range(%ld, %ld)\n", + (long)offset, (long)size_to_read, (long)readRangeEvent->offset, + (long)readRangeEvent->size_to_read); + exchange->error = true; + return 0; + } + exchange->nextEvent++; + if (!readRangeEvent->errorMsg.empty()) { + snprintf(out_error_string, error_string_max_size, "%s", + readRangeEvent->errorMsg.c_str()); + return 0; + } + memcpy(buffer, readRangeEvent->response.data(), + readRangeEvent->response.size()); + return readRangeEvent->response.size(); +} + +TEST(networking, custom) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/my.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "dummy"; // dummy value: not used + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/my.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<ReadRangeEvent> event(new ReadRangeEvent()); + event->ctx = ctx; + event->offset = 3670016; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 2.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = -49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 2.25); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + // Once again ! No network access + + P = proj_create(ctx, + "+proj=vgridshift +grids=https://foo/my.tif +multiplier=1"); + ASSERT_NE(P, nullptr); + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, getfilesize) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/getfilesize.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/test_vgrid_single_strip_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 550, f), 550U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "bytes 0-16383/4153510"; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_destroy(P); + + P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/getfilesize.tif +multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_destroy(P); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, simul_open_error) { + auto ctx = proj_context_create(); + proj_log_func(ctx, nullptr, silent_logger); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/open_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->errorMsg = "Cannot open file"; + event->file_id = 1; + + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create( + ctx, + "+proj=vgridshift +grids=https://foo/open_error.tif +multiplier=1"); + + ASSERT_EQ(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +TEST(networking, simul_read_range_error) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + ExchangeWithCallback exchange; + ASSERT_TRUE(proj_context_set_network_callbacks(ctx, open_cbk, close_cbk, + get_header_value_cbk, + read_range_cbk, &exchange)); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 0; + event->size_to_read = 16384; + event->response.resize(16384); + event->file_id = 1; + + const char *proj_source_data = getenv("PROJ_SOURCE_DATA"); + ASSERT_TRUE(proj_source_data != nullptr); + std::string filename(proj_source_data); + filename += "/tests/egm96_15_uncompressed_truncated.tif"; + FILE *f = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(f != nullptr); + ASSERT_EQ(fread(&event->response[0], 1, 956, f), 956U); + fclose(f); + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<GetHeaderValueEvent> event(new GetHeaderValueEvent()); + event->ctx = ctx; + event->key = "Content-Range"; + event->value = "dummy"; // dummy value: not used + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 1; + exchange.events.emplace_back(std::move(event)); + } + + auto P = proj_create(ctx, "+proj=vgridshift " + "+grids=https://foo/read_range_error.tif " + "+multiplier=1"); + + ASSERT_NE(P, nullptr); + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<OpenEvent> event(new OpenEvent()); + event->ctx = ctx; + event->url = "https://foo/read_range_error.tif"; + event->offset = 524288; + event->size_to_read = 278528; + event->response.resize(278528); + event->file_id = 2; + float f = 1.25; + for (size_t i = 0; i < 278528 / sizeof(float); i++) { + memcpy(&event->response[i * sizeof(float)], &f, sizeof(float)); + } + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = 49 / 180. * M_PI; + double z = 0; + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, 1.25); + } + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + { + std::unique_ptr<ReadRangeEvent> event(new ReadRangeEvent()); + event->ctx = ctx; + event->offset = 3670016; + event->size_to_read = 278528; + event->errorMsg = "read range error"; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + + { + double lon = 2 / 180. * M_PI; + double lat = -49 / 180. * M_PI; + double z = 0; + proj_log_func(ctx, nullptr, silent_logger); + ASSERT_EQ(proj_trans_generic(P, PJ_FWD, &lon, sizeof(double), 1, &lat, + sizeof(double), 1, &z, sizeof(double), 1, + nullptr, 0, 0), + 1U); + EXPECT_EQ(z, HUGE_VAL); + } + { + std::unique_ptr<CloseEvent> event(new CloseEvent()); + event->ctx = ctx; + event->file_id = 2; + exchange.events.emplace_back(std::move(event)); + } + proj_destroy(P); + + ASSERT_TRUE(exchange.allConsumedAndNoError()); + + proj_context_destroy(ctx); +} + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_hgridshift) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_assign_context(P, ctx); // (dummy) test context reassignment + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, 39.99999839, 1e-8); + EXPECT_NEAR(c.xyz.y, -79.99999807, 1e-8); + EXPECT_NEAR(c.xyz.z, 0, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_vgridshift) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // WGS84 to EGM2008 height. Using egm08_25.tif + auto P = + proj_create_crs_to_crs(ctx, "EPSG:4326", "EPSG:4326+3855", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = -30; // lat + c.xyz.y = 150; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_assign_context(P, ctx); // (dummy) test context reassignment + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, -30, 1e-8); + EXPECT_NEAR(c.xyz.y, 150, 1e-8); + EXPECT_NEAR(c.xyz.z, -31.89, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, curl_vgridshift_vertcon) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // NGVD29 to NAVD88 height. Using vertcone.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269+7968", "EPSG:4269+5703", + nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_NEAR(c.xyz.x, 40, 1e-8); + EXPECT_NEAR(c.xyz.y, -80, 1e-8); + EXPECT_NEAR(c.xyz.z, -0.15, 1e-2); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, network_endpoint_env_variable) { + putenv(const_cast<char *>("PROJ_NETWORK_ENDPOINT=http://0.0.0.0/")); + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + putenv(const_cast<char *>("PROJ_NETWORK_ENDPOINT=")); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_EQ(c.xyz.x, HUGE_VAL); +} + +#endif + +// --------------------------------------------------------------------------- + +#ifdef CURL_ENABLED + +TEST(networking, network_endpoint_api) { + auto ctx = proj_context_create(); + proj_context_set_enable_network(ctx, true); + proj_context_set_url_endpoint(ctx, "http://0.0.0.0"); + + // NAD83 to NAD83(HARN) in West-Virginia. Using wvhpgn.tif + auto P = proj_create_crs_to_crs(ctx, "EPSG:4269", "EPSG:4152", nullptr); + ASSERT_NE(P, nullptr); + + PJ_COORD c; + c.xyz.x = 40; // lat + c.xyz.y = -80; // lon + c.xyz.z = 0; + c = proj_trans(P, PJ_FWD, c); + + proj_destroy(P); + proj_context_destroy(ctx); + + EXPECT_EQ(c.xyz.x, HUGE_VAL); +} + +#endif + +} // namespace diff --git a/test/unit/test_operation.cpp b/test/unit/test_operation.cpp index 9cde8822..753878c5 100644 --- a/test/unit/test_operation.cpp +++ b/test/unit/test_operation.cpp @@ -4789,7 +4789,7 @@ TEST(operation, geogCRS_to_geogCRS_context_concatenated_operation) { EXPECT_TRUE(nn_dynamic_pointer_cast<ConcatenatedOperation>(list[0]) != nullptr); - auto grids = list[0]->gridsNeeded(DatabaseContext::create()); + auto grids = list[0]->gridsNeeded(DatabaseContext::create(), false); EXPECT_EQ(grids.size(), 1U); } @@ -6402,7 +6402,7 @@ TEST(operation, transformation_height_to_PROJ_string) { EXPECT_EQ(transf->exportToPROJString(PROJStringFormatter::create().get()), "+proj=vgridshift +grids=egm08_25.gtx +multiplier=1"); - auto grids = transf->gridsNeeded(DatabaseContext::create()); + auto grids = transf->gridsNeeded(DatabaseContext::create(), false); ASSERT_EQ(grids.size(), 1U); auto gridDesc = *(grids.begin()); EXPECT_EQ(gridDesc.shortName, "egm08_25.gtx"); @@ -6702,7 +6702,7 @@ TEST(operation, compoundCRS_with_boundGeogCRS_and_boundVerticalCRS_to_geogCRS) { "+step +proj=unitconvert +xy_in=rad +xy_out=deg " "+step +proj=axisswap +order=2,1"); - auto grids = op->gridsNeeded(DatabaseContext::create()); + auto grids = op->gridsNeeded(DatabaseContext::create(), false); EXPECT_EQ(grids.size(), 1U); auto opInverse = CoordinateOperationFactory::create()->createOperation( @@ -8128,8 +8128,8 @@ TEST(operation, isPROJInstantiable) { auto transformation = Transformation::createGeocentricTranslations( PropertyMap(), GeographicCRS::EPSG_4269, GeographicCRS::EPSG_4326, 1.0, 2.0, 3.0, {}); - EXPECT_TRUE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_TRUE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } // Missing grid @@ -8137,8 +8137,8 @@ TEST(operation, isPROJInstantiable) { auto transformation = Transformation::createNTv2( PropertyMap(), GeographicCRS::EPSG_4807, GeographicCRS::EPSG_4326, "foo.gsb", std::vector<PositionalAccuracyNNPtr>()); - EXPECT_FALSE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_FALSE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } // Unsupported method @@ -8149,8 +8149,8 @@ TEST(operation, isPROJInstantiable) { PropertyMap(), std::vector<OperationParameterNNPtr>{}), std::vector<GeneralParameterValueNNPtr>{}, std::vector<PositionalAccuracyNNPtr>{}); - EXPECT_FALSE( - transformation->isPROJInstantiable(DatabaseContext::create())); + EXPECT_FALSE(transformation->isPROJInstantiable( + DatabaseContext::create(), false)); } } diff --git a/travis/csa/before_install.sh b/travis/csa/before_install.sh index c2e29064..dcb7733b 100755 --- a/travis/csa/before_install.sh +++ b/travis/csa/before_install.sh @@ -5,7 +5,7 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev CLANG_LLVM=clang+llvm-6.0.0-x86_64-linux-gnu-ubuntu-16.04 wget http://releases.llvm.org/6.0.0/$CLANG_LLVM.tar.xz diff --git a/travis/linux_clang/before_install.sh b/travis/linux_clang/before_install.sh index 8ce465a2..c4b8acad 100755 --- a/travis/linux_clang/before_install.sh +++ b/travis/linux_clang/before_install.sh @@ -5,4 +5,4 @@ set -e ./travis/before_install_apt.sh ./travis/before_install_pip.sh -sudo apt-get install -qq sqlite3 libsqlite3-dev +sudo apt-get install -qq sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev diff --git a/travis/linux_gcc/before_install.sh b/travis/linux_gcc/before_install.sh index 7725b3ef..9db2ea89 100755 --- a/travis/linux_gcc/before_install.sh +++ b/travis/linux_gcc/before_install.sh @@ -9,6 +9,7 @@ sudo apt-get install -qq \ lcov \ doxygen graphviz \ sqlite3 libsqlite3-dev \ + libtiff-dev libcurl4-openssl-dev \ cppcheck scripts/cppcheck.sh diff --git a/travis/linux_gcc7/before_install.sh b/travis/linux_gcc7/before_install.sh index a3f6c8c0..2ef2feee 100755 --- a/travis/linux_gcc7/before_install.sh +++ b/travis/linux_gcc7/before_install.sh @@ -8,7 +8,8 @@ set -e sudo apt-get install -qq \ lcov \ doxygen graphviz \ - sqlite3 libsqlite3-dev + sqlite3 libsqlite3-dev \ + libtiff-dev libcurl4-openssl-dev #scripts/cppcheck.sh #scripts/doxygen.sh diff --git a/travis/mingw32/install.sh b/travis/mingw32/install.sh index 52e4f7a6..abe79c7b 100755 --- a/travis/mingw32/install.sh +++ b/travis/mingw32/install.sh @@ -2,6 +2,9 @@ set -e +unset CC +unset CXX + export CCACHE_CPP2=yes export PROJ_DB_CACHE_DIR="$HOME/.ccache" @@ -16,6 +19,22 @@ ln -s $MINGW_PREFIX/libgcc_s_seh-1.dll $WINE_SYSDIR ln -s $MINGW_PREFIX/libgcc_s_sjlj-1.dll $WINE_SYSDIR ln -s /usr/$MINGW_ARCH/lib/libwinpthread-1.dll $WINE_SYSDIR +# build zlib +wget https://github.com/madler/zlib/archive/v1.2.11.tar.gz +tar xzf v1.2.11.tar.gz +(cd zlib-1.2.11 && sudo make install -fwin32/Makefile.gcc SHARED_MODE=1 PREFIX=x86_64-w64-mingw32- DESTDIR=/usr/$MINGW_ARCH/) +sudo mkdir -p /usr/$MINGW_ARCH/include +sudo mkdir -p /usr/$MINGW_ARCH/lib +sudo cp /usr/$MINGW_ARCH/*.h /usr/$MINGW_ARCH/include +sudo cp /usr/$MINGW_ARCH/libz.* /usr/$MINGW_ARCH/lib +ln -s /usr/$MINGW_ARCH/zlib1.dll $WINE_SYSDIR + +# build libtiff +wget https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz +tar xzf tiff-4.1.0.tar.gz +(cd tiff-4.1.0 && ./configure --host=$MINGW_ARCH --prefix=/usr/$MINGW_ARCH && make -j2 && sudo make install) +ln -s /usr/$MINGW_ARCH/bin/libtiff-5.dll $WINE_SYSDIR + # build sqlite3 wget https://sqlite.org/2018/sqlite-autoconf-3250100.tar.gz tar xzf sqlite-autoconf-3250100.tar.gz @@ -28,7 +47,7 @@ ln -s /usr/$MINGW_ARCH/bin/libsqlite3-0.dll $WINE_SYSDIR # autoconf build mkdir build_autoconf cd build_autoconf -CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install +CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install --without-curl make -j2 make install make dist-all @@ -44,7 +63,7 @@ cd .. # autoconf build with grids mkdir build_autoconf_grids cd build_autoconf_grids -CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install_nad +CC="ccache $MINGW_ARCH-gcc" CXX="ccache $MINGW_ARCH-g++" LD=$MINGW_ARCH-ld ../configure --host=$MINGW_ARCH --prefix=/tmp/proj_autoconf_install_nad --without-curl make -j2 make install find /tmp/proj_autoconf_install_nad diff --git a/travis/osx/before_install.sh b/travis/osx/before_install.sh index 09a7a74f..e457ec8c 100755 --- a/travis/osx/before_install.sh +++ b/travis/osx/before_install.sh @@ -6,10 +6,11 @@ export PATH=$HOME/Library/Python/3.7/bin:$PATH brew update brew install ccache -brew install sqlite3 +#brew upgrade sqlite3 +#brew upgrade libtiff brew install doxygen -brew install md5sha1sum -brew reinstall python +#brew install md5sha1sum +#brew reinstall python brew reinstall wget ./travis/before_install_pip.sh |